org.apache.kylin.common.persistence.ResourceStore类的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(10.6k)|赞(0)|评价(0)|浏览(107)

本文整理了Java中org.apache.kylin.common.persistence.ResourceStore类的一些代码示例，展示了ResourceStore类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。ResourceStore类的具体详情如下：
包路径：org.apache.kylin.common.persistence.ResourceStore
类名称：ResourceStore

ResourceStore介绍

[英]A general purpose resource store to persist small metadata, like JSON files. In additional to raw bytes save and load, the store takes special care for concurrent modifications by using a timestamp based test-and-set mechanism to detect (and refuse) dirty writes.
[中]一个通用资源存储区，用于保存小型元数据，如JSON文件。除了原始字节的保存和加载，存储还特别注意并发修改，使用基于时间戳的测试和设置机制来检测（并拒绝）脏写。

代码示例

代码示例来源：origin: apache/kylin

public static void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException {
  long startTime = System.currentTimeMillis();
  ResourceStore from = ResourceStore.getStore(kylinConfig);
  KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
  ResourceStore to = ResourceStore.getStore(localConfig);
  for (String path : dumpList) {
    RawResource res = from.getResource(path);
    if (res == null)
      throw new IllegalStateException("No resource found at -- " + path);
    to.putResource(path, res.content(), res.lastModified());
    res.content().close();
  }
  logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime);
}

代码示例来源：origin: apache/kylin

public void removeSnapshot(String resourcePath) throws IOException {
  ResourceStore store = getStore();
  store.deleteResource(resourcePath);
  snapshotCache.invalidate(resourcePath);
}

代码示例来源：origin: apache/kylin

public NavigableSet<String> list(KylinConfig config, String path) throws IOException {
  ResourceStore store = ResourceStore.getStore(config);
  NavigableSet<String> result = store.listResources(path);
  System.out.println("" + result);
  return result;
}

代码示例来源：origin: apache/kylin

public String getMetaStoreUUID() throws IOException {
  if (!exists(ResourceStore.METASTORE_UUID_TAG)) {
    checkAndPutResource(ResourceStore.METASTORE_UUID_TAG, new StringEntity(createMetaStoreUUID()), 0,
        StringEntity.serializer);
  }
  StringEntity entity = getResource(ResourceStore.METASTORE_UUID_TAG, StringEntity.serializer);
  return entity.toString();
}

代码示例来源：origin: apache/kylin

public static String getMetaStoreId() throws IOException {
  KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
  ResourceStore store = ResourceStore.getStore(kylinConfig);
  return store.getMetaStoreUUID();
}

代码示例来源：origin: apache/kylin

public static void appendFactTableData(String factTableContent, String factTableName) throws IOException {
  // Write to resource store
  ResourceStore store = ResourceStore.getStore(config());
  InputStream in = new ByteArrayInputStream(factTableContent.getBytes("UTF-8"));
  String factTablePath = "/data/" + factTableName + ".csv";
  File tmpFile = File.createTempFile(factTableName, "csv");
  FileOutputStream out = new FileOutputStream(tmpFile);
  InputStream tempIn = null;
  try {
    if (store.exists(factTablePath)) {
      InputStream oldContent = store.getResource(factTablePath).content();
      IOUtils.copy(oldContent, out);
    }
    IOUtils.copy(in, out);
    IOUtils.closeQuietly(in);
    IOUtils.closeQuietly(out);
    store.deleteResource(factTablePath);
    tempIn = new FileInputStream(tmpFile);
    store.putResource(factTablePath, tempIn, System.currentTimeMillis());
  } finally {
    IOUtils.closeQuietly(out);
    IOUtils.closeQuietly(in);
    IOUtils.closeQuietly(tempIn);
  }
}

代码示例来源：origin: apache/kylin

@Test
public void testRollback() throws Exception {
  ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
  byte[] bytes = new byte[] { 0, 1, 2 };
  RawResource raw;
  Checkpoint cp;
  cp = store.checkpoint();
  try {
    store.putResource("/res1", new StringEntity("data1"), 1000, StringEntity.serializer);
  } finally {
    cp.close();
  StringEntity str = store.getResource("/res1", StringEntity.serializer);
  assertEquals("data1", str.toString());
  cp = store.checkpoint();
  try {
    ByteArrayInputStream is = new ByteArrayInputStream(bytes);
    store.putResource("/res2", is, 2000);
    is.close();
    store.putResource("/res1", str, 2000, StringEntity.serializer);
    store.deleteResource("/res1");
    assertEquals(null, store.getResource("/res1"));
    assertEquals(2000, (raw = store.getResource("/res2")).lastModified());
    raw.content().close();
    assertEquals(null, store.getResource("/res2"));

代码示例来源：origin: apache/kylin

store.deleteResource(path1);
store.deleteResource(path2);
store.checkAndPutResource(path1, content1, StringEntity.serializer);
assertTrue(store.exists(path1));
t = store.getResource(path1, StringEntity.serializer);
assertEquals(content1, t);
store.checkAndPutResource(path2, content2, StringEntity.serializer);
assertTrue(store.exists(path2));
t = store.getResource(path2, StringEntity.serializer);
assertEquals(content2, t);
store.checkAndPutResource(path2, t, StringEntity.serializer);
  store.checkAndPutResource(path2, t, StringEntity.serializer);
  fail("write conflict should trigger IllegalStateException");
} catch (WriteConflictException e) {
list = store.listResources(dir1);
System.out.println(list);
assertTrue(list.contains(path1));
assertTrue(list.contains(path2) == false);
list = store.listResources(dir2);
assertTrue(list.contains(path2));
assertTrue(list.contains(path1) == false);
list = store.listResources("/");

代码示例来源：origin: apache/kylin

private ExecutablePO readJobResource(String path) throws IOException {
  return store.getResource(path, JOB_SERIALIZER);
}

代码示例来源：origin: apache/kylin

public void remove(KylinConfig config, String path) throws IOException {
    ResourceStore store = ResourceStore.getStore(config);
    resetR(store, path);
  }
}

代码示例来源：origin: apache/kylin

children = src.listResources(path);
    RawResource res = src.getResource(path);
    if (res != null) {
      try {
        dst.putResource(path, res.content(), res.lastModified());
      } finally {
        res.close();

代码示例来源：origin: apache/kylin

public String cat(KylinConfig config, String path) throws IOException {
  ResourceStore store = ResourceStore.getStore(config);
  StringBuffer sb = new StringBuffer();
  String line;
  try (InputStream is = store.getResource(path).content();
      BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
    while ((line = br.readLine()) != null) {
      System.out.println(line);
      sb.append(line).append('\n');
    }
  }
  return sb.toString();
}

代码示例来源：origin: apache/kylin

public void refresh() throws IOException {
  ObjectMapper mapper = new ObjectMapper();
  mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
  List<String> all = Lists.newArrayList();
  collectFiles(this.store, "/", all);
  for (String path : all) {
    if (path.endsWith(MetadataConstants.FILE_SURFIX) && !(path.startsWith(ResourceStore.DICT_RESOURCE_ROOT) || path.startsWith(ResourceStore.SNAPSHOT_RESOURCE_ROOT))) {
      logger.info("Updating metadata version of path {}", path);
      ObjectNode objectNode = (ObjectNode) mapper.readTree(this.store.getResource(path).content());
      objectNode.put("version", version);
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      mapper.writeValue(baos, objectNode);
      ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
      this.store.putResource(path, bais, System.currentTimeMillis());
    }
  }
}

代码示例来源：origin: apache/kylin

private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
  Set<String> dumpList = new LinkedHashSet<>();
  dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
  ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
  for (CubeSegment segment : segments) {
    dumpList.addAll(segment.getDictionaryPaths());
    if (rs.exists(segment.getStatisticsResourcePath())) {
      // cube statistics is not available for new segment
      dumpList.add(segment.getStatisticsResourcePath());
    }
  }
  JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
      this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}

代码示例来源：origin: apache/kylin

@Test
public void testGetAllCubes() throws Exception {
  final ResourceStore store = ResourceStore.getStore(getTestConfig());
  final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
  assertTrue(cubePath.size() > 1);
  final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT,
      CubeManager.CUBE_SERIALIZER);
  assertEquals(cubePath.size(), cubes.size());
}

代码示例来源：origin: apache/kylin

private static long testWritePerformance(ResourceStore store) throws IOException {
  store.deleteResource(PERFORMANCE_TEST_ROOT_PATH);
  StringEntity content = new StringEntity("something");
  long startTime = System.currentTimeMillis();
  for (int i = 0; i < TEST_RESOURCE_COUNT; i++) {
    String resourcePath = PERFORMANCE_TEST_ROOT_PATH + "/res_" + i;
    store.putResource(resourcePath, content, 0, StringEntity.serializer);
  }
  return System.currentTimeMillis() - startTime;
}

代码示例来源：origin: apache/kylin

public void saveQuery(final String creator, final Query query) throws IOException {
  List<Query> queries = getQueries(creator);
  queries.add(query);
  Query[] queryArray = new Query[queries.size()];
  QueryRecord record = new QueryRecord(queries.toArray(queryArray));
  queryStore.putResource(getQueryKeyById(creator), record, System.currentTimeMillis(),
      QueryRecordSerializer.getInstance());
  return;
}

代码示例来源：origin: apache/kylin

public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException {
  try (AutoLock lock = srcExtMapLock.lockForWrite()) {
    if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
      throw new IllegalArgumentException();
    }
    // updating a legacy global table
    if (tableExt.getProject() == null) {
      if (getTableExt(tableExt.getIdentity(), prj).getProject() != null)
        throw new IllegalStateException(
            "Updating a legacy global TableExtDesc while a project level version exists: "
                + tableExt.getIdentity() + ", " + prj);
      prj = tableExt.getProject();
    }
    tableExt.init(prj);
    // what is this doing??
    String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj);
    ResourceStore store = getStore();
    TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER);
    if (t != null && t.getIdentity() == null)
      store.deleteResource(path);
    srcExtCrud.save(tableExt);
  }
}

代码示例来源：origin: apache/kylin

public List<Draft> list(String project) throws IOException {
  List<Draft> result = new ArrayList<>();
  ResourceStore store = getStore();
  NavigableSet<String> listPath = store.listResources(ResourceStore.DRAFT_RESOURCE_ROOT);
  if (listPath == null)
    return result;
  
  for (String path : listPath) {
    Draft draft = store.getResource(path, DRAFT_SERIALIZER);
    
    if (draft == null)
      continue;
    
    if (project == null || project.equals(draft.getProject()))
      result.add(draft);
  }
  return result;
}

代码示例来源：origin: apache/kylin

public void reloadAll() throws IOException {
  logger.debug("Reloading " + entityType.getSimpleName() + " from " + store.getReadableResourcePath(resRootPath));
  cache.clear();
  List<String> paths = store.collectResourceRecursively(resRootPath, resPathSuffix);
  for (String path : paths) {
    reloadQuietlyAt(path);
  }
  logger.debug("Loaded " + cache.size() + " " + entityType.getSimpleName() + "(s) out of " + paths.size()
      + " resource");
}

内容来源于网络，如有侵权，请联系作者删除！

ResourceStore

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com