io.prestosql.memory.QueryContext类的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(13.4k)|赞(0)|评价(0)|浏览(95)

本文整理了Java中io.prestosql.memory.QueryContext类的一些代码示例，展示了QueryContext类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。QueryContext类的具体详情如下：
包路径：io.prestosql.memory.QueryContext
类名称：QueryContext

QueryContext介绍

[英]This interface and the LegacyQueryContext implementation are written to ease the removal of the system memory pool. Once the system pool is removed they should both be removed.
[中]编写此接口和LegacyQueryContext实现是为了简化系统内存池的删除。一旦系统池被移除，它们都应该被移除。

代码示例

代码示例来源：origin: io.prestosql/presto-main

private QueryContext createQueryContext(
    QueryId queryId,
    LocalMemoryManager localMemoryManager,
    NodeMemoryConfig nodeMemoryConfig,
    LocalSpillManager localSpillManager,
    GcMonitor gcMonitor,
    DataSize maxQueryUserMemoryPerNode,
    DataSize maxQueryTotalMemoryPerNode,
    DataSize maxQuerySpillPerNode)
{
  return new QueryContext(
      queryId,
      maxQueryUserMemoryPerNode,
      maxQueryTotalMemoryPerNode,
      localMemoryManager.getGeneralPool(),
      gcMonitor,
      taskNotificationExecutor,
      driverYieldExecutor,
      maxQuerySpillPerNode,
      localSpillManager.getSpillSpaceTracker());
}

代码示例来源：origin: io.prestosql/presto-main

private static TaskContext createTaskContext(QueryContext queryContext, Session session, TaskStateMachine taskStateMachine)
{
  return queryContext.addTaskContext(
      taskStateMachine,
      session,
      true,
      true,
      OptionalInt.empty());
}

代码示例来源：origin: prestosql/presto

private long getMemoryAlreadyBeingRevoked(Collection<SqlTask> sqlTasks, MemoryPool memoryPool)
{
  return sqlTasks.stream()
      .filter(task -> task.getTaskStatus().getState() == TaskState.RUNNING)
      .filter(task -> task.getQueryContext().getMemoryPool() == memoryPool)
      .mapToLong(task -> task.getQueryContext().accept(new TraversingQueryContextVisitor<Void, Long>()
      {
        @Override
        public Long visitOperatorContext(OperatorContext operatorContext, Void context)
        {
          if (operatorContext.isMemoryRevokingRequested()) {
            return operatorContext.getReservedRevocableBytes();
          }
          return 0L;
        }
        @Override
        public Long mergeResults(List<Long> childrenResults)
        {
          return childrenResults.stream()
              .mapToLong(i -> i).sum();
        }
      }, null))
      .sum();
}

代码示例来源：origin: io.prestosql/presto-benchmark

SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
TaskContext taskContext = new QueryContext(
    new QueryId("test"),
    new DataSize(256, MEGABYTE),
    new DataSize(256, MEGABYTE),
    spillSpaceTracker)
    .addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()),
        session,
        false,

代码示例来源：origin: io.prestosql/presto-main

QueryContext queryContext = new QueryContext(
    new QueryId("query"),
    new DataSize(10, BYTE),
queryContext.getQueryMemoryContext().initializeLocalMemoryContexts("test");
LocalMemoryContext userMemoryContext = queryContext.getQueryMemoryContext().localUserMemoryContext();
LocalMemoryContext revocableMemoryContext = queryContext.getQueryMemoryContext().localRevocableMemoryContext();
assertTrue(userMemoryContext.setBytes(3).isDone());
assertTrue(revocableMemoryContext.setBytes(5).isDone());
queryContext.setMemoryPool(reservedPool);

代码示例来源：origin: io.prestosql/presto-main

@Override
public synchronized void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments)
{
  if (coordinatorId != null && coordinatorId.equals(assignments.getCoordinatorId()) && assignments.getVersion() <= currentMemoryPoolAssignmentVersion) {
    return;
  }
  currentMemoryPoolAssignmentVersion = assignments.getVersion();
  if (coordinatorId != null && !coordinatorId.equals(assignments.getCoordinatorId())) {
    log.warn("Switching coordinator affinity from " + coordinatorId + " to " + assignments.getCoordinatorId());
  }
  coordinatorId = assignments.getCoordinatorId();
  for (MemoryPoolAssignment assignment : assignments.getAssignments()) {
    if (assignment.getPoolId().equals(GENERAL_POOL)) {
      queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(localMemoryManager.getGeneralPool());
    }
    else if (assignment.getPoolId().equals(RESERVED_POOL)) {
      MemoryPool reservedPool = localMemoryManager.getReservedPool()
          .orElseThrow(() -> new IllegalArgumentException(format("Cannot move %s to the reserved pool as the reserved pool is not enabled", assignment.getQueryId())));
      queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(reservedPool);
    }
    else {
      new IllegalArgumentException(format("Cannot move %s to %s as the target memory pool id is invalid", assignment.getQueryId(), assignment.getPoolId()));
    }
  }
}

代码示例来源：origin: prestosql/presto

@BeforeMethod
public void setUp()
{
  executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s"));
  scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
  TaskContext taskContext = TestingTaskContext.builder(executor, scheduledExecutor, TEST_SESSION)
      .setQueryMaxMemory(DataSize.valueOf("100MB"))
      .setMemoryPoolSize(DataSize.valueOf("10B"))
      .setQueryId(QUERY_ID)
      .build();
  memoryPool = taskContext.getQueryContext().getMemoryPool();
  driverContext = taskContext
      .addPipelineContext(0, true, true, false)
      .addDriverContext();
}

代码示例来源：origin: io.prestosql/presto-main

@Override
public R visitQueryContext(QueryContext queryContext, C visitContext)
{
  return mergeResults(queryContext.acceptChildren(this, visitContext));
}

代码示例来源：origin: io.prestosql/presto-main

private synchronized ListenableFuture<?> updateSystemMemory(String allocationTag, long delta)
{
  // We call memoryPool.getQueryMemoryReservation(queryId) instead of calling queryMemoryContext.getUserMemory() to
  // calculate the total memory size.
  //
  // Calling the latter can result in a deadlock:
  // * A thread doing a user allocation will acquire locks in this order:
  //   1. monitor of queryMemoryContext.userAggregateMemoryContext
  //   2. monitor of this (QueryContext)
  // * The current thread doing a system allocation will acquire locks in this order:
  //   1. monitor of this (QueryContext)
  //   2. monitor of queryMemoryContext.userAggregateMemoryContext
  // Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory
  // calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. For concurrent
  // allocations of the same type (e.g., tryUpdateUserMemory/updateUserMemory) it is not possible as they share
  // the same RootAggregatedMemoryContext instance, and one of the threads will be blocked on the monitor of that
  // RootAggregatedMemoryContext instance even before calling the QueryContext methods (the monitors of
  // RootAggregatedMemoryContext instance and this will be acquired in the same order).
  long totalMemory = memoryPool.getQueryMemoryReservation(queryId);
  if (delta >= 0) {
    enforceTotalMemoryLimit(totalMemory, delta, maxTotalMemory);
    return memoryPool.reserve(queryId, allocationTag, delta);
  }
  memoryPool.free(queryId, allocationTag, -delta);
  return NOT_BLOCKED;
}

代码示例来源：origin: io.prestosql/presto-main

@BeforeMethod
public void setUpTest()
{
  memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize);
  queryContext = new QueryContext(
      new QueryId("test_query"),
      queryMaxMemory,
      queryMaxTotalMemory,
      memoryPool,
      new TestingGcMonitor(),
      notificationExecutor,
      yieldExecutor,
      queryMaxSpillSize,
      spillSpaceTracker);
  taskContext = queryContext.addTaskContext(
      new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor),
      testSessionBuilder().build(),
      true,
      true,
      OptionalInt.empty());
  pipelineContext = taskContext.addPipelineContext(0, true, true, false);
  driverContext = pipelineContext.addDriverContext();
  operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator");
}

代码示例来源：origin: prestosql/presto

@Override
public synchronized void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments)
{
  if (coordinatorId != null && coordinatorId.equals(assignments.getCoordinatorId()) && assignments.getVersion() <= currentMemoryPoolAssignmentVersion) {
    return;
  }
  currentMemoryPoolAssignmentVersion = assignments.getVersion();
  if (coordinatorId != null && !coordinatorId.equals(assignments.getCoordinatorId())) {
    log.warn("Switching coordinator affinity from " + coordinatorId + " to " + assignments.getCoordinatorId());
  }
  coordinatorId = assignments.getCoordinatorId();
  for (MemoryPoolAssignment assignment : assignments.getAssignments()) {
    if (assignment.getPoolId().equals(GENERAL_POOL)) {
      queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(localMemoryManager.getGeneralPool());
    }
    else if (assignment.getPoolId().equals(RESERVED_POOL)) {
      MemoryPool reservedPool = localMemoryManager.getReservedPool()
          .orElseThrow(() -> new IllegalArgumentException(format("Cannot move %s to the reserved pool as the reserved pool is not enabled", assignment.getQueryId())));
      queryContexts.getUnchecked(assignment.getQueryId()).setMemoryPool(reservedPool);
    }
    else {
      new IllegalArgumentException(format("Cannot move %s to %s as the target memory pool id is invalid", assignment.getQueryId(), assignment.getPoolId()));
    }
  }
}

代码示例来源：origin: io.prestosql/presto-main

@BeforeMethod
public void setUp()
{
  executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s"));
  scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
  TaskContext taskContext = TestingTaskContext.builder(executor, scheduledExecutor, TEST_SESSION)
      .setQueryMaxMemory(DataSize.valueOf("100MB"))
      .setMemoryPoolSize(DataSize.valueOf("10B"))
      .setQueryId(QUERY_ID)
      .build();
  memoryPool = taskContext.getQueryContext().getMemoryPool();
  driverContext = taskContext
      .addPipelineContext(0, true, true, false)
      .addDriverContext();
}

代码示例来源：origin: prestosql/presto

@Override
public R visitQueryContext(QueryContext queryContext, C visitContext)
{
  return mergeResults(queryContext.acceptChildren(this, visitContext));
}

代码示例来源：origin: io.prestosql/presto-main

private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificationExecutor, ScheduledExecutorService driverYieldExecutor, TaskStateMachine taskStateMachine)
{
  QueryContext queryContext = new QueryContext(
      new QueryId("queryid"),
      new DataSize(1, MEGABYTE),
      new DataSize(2, MEGABYTE),
      new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)),
      new TestingGcMonitor(),
      taskNotificationExecutor,
      driverYieldExecutor,
      new DataSize(1, MEGABYTE),
      new SpillSpaceTracker(new DataSize(1, GIGABYTE)));
  return queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty());
}

代码示例来源：origin: io.prestosql/presto-main

private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, OutputBuffers outputBuffers)
{
  sqlTaskManager.getQueryContext(taskId.getQueryId())
      .addTaskContext(new TaskStateMachine(taskId, directExecutor()), testSessionBuilder().build(), false, false, OptionalInt.empty());
  return sqlTaskManager.updateTask(TEST_SESSION,
      taskId,
      Optional.of(PLAN_FRAGMENT),
      ImmutableList.of(),
      outputBuffers,
      OptionalInt.empty());
}

代码示例来源：origin: io.prestosql/presto-main

public TaskContext build()
  {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize);
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(maxSpillSize);
    QueryContext queryContext = new QueryContext(
        queryId,
        queryMaxMemory,
        queryMaxTotalMemory,
        memoryPool,
        GC_MONITOR,
        notificationExecutor,
        yieldExecutor,
        queryMaxSpillSize,
        spillSpaceTracker);
    return createTaskContext(queryContext, session, taskStateMachine);
  }
}

代码示例来源：origin: io.prestosql/presto-main

private long getMemoryAlreadyBeingRevoked(Collection<SqlTask> sqlTasks, MemoryPool memoryPool)
{
  return sqlTasks.stream()
      .filter(task -> task.getTaskStatus().getState() == TaskState.RUNNING)
      .filter(task -> task.getQueryContext().getMemoryPool() == memoryPool)
      .mapToLong(task -> task.getQueryContext().accept(new TraversingQueryContextVisitor<Void, Long>()
      {
        @Override
        public Long visitOperatorContext(OperatorContext operatorContext, Void context)
        {
          if (operatorContext.isMemoryRevokingRequested()) {
            return operatorContext.getReservedRevocableBytes();
          }
          return 0L;
        }
        @Override
        public Long mergeResults(List<Long> childrenResults)
        {
          return childrenResults.stream()
              .mapToLong(i -> i).sum();
        }
      }, null))
      .sum();
}

代码示例来源：origin: prestosql/presto

@Test
public void testCountAlreadyRevokedMemoryWithinAPool()
    throws Exception
{
  // Given
  SqlTask sqlTask1 = newSqlTask();
  MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, BYTE));
  sqlTask1.getQueryContext().setMemoryPool(anotherMemoryPool);
  OperatorContext operatorContext1 = createContexts(sqlTask1);
  SqlTask sqlTask2 = newSqlTask();
  OperatorContext operatorContext2 = createContexts(sqlTask2);
  List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
  MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, executor, 1.0, 1.0);
  allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);
  /*
   * sqlTask1 fills its pool
   */
  operatorContext1.localRevocableMemoryContext().setBytes(12);
  requestMemoryRevoking(scheduler);
  assertMemoryRevokingRequestedFor(operatorContext1);
  /*
   * When sqlTask2 fills its pool
   */
  operatorContext2.localRevocableMemoryContext().setBytes(12);
  requestMemoryRevoking(scheduler);
  /*
   * Then sqlTask2 should be asked to revoke its memory too
   */
  assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
}

代码示例来源：origin: io.prestosql/presto-main

QueryContext queryContext = new QueryContext(taskId.getQueryId(),
    new DataSize(1, MEGABYTE),
    new DataSize(2, MEGABYTE),
    new DataSize(1, MEGABYTE),
    spillSpaceTracker);
this.taskContext = queryContext.addTaskContext(taskStateMachine, TEST_SESSION, true, true, totalPartitions);

代码示例来源：origin: prestosql/presto

private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, OutputBuffers outputBuffers)
{
  sqlTaskManager.getQueryContext(taskId.getQueryId())
      .addTaskContext(new TaskStateMachine(taskId, directExecutor()), testSessionBuilder().build(), false, false, OptionalInt.empty());
  return sqlTaskManager.updateTask(TEST_SESSION,
      taskId,
      Optional.of(PLAN_FRAGMENT),
      ImmutableList.of(),
      outputBuffers,
      OptionalInt.empty());
}

内容来源于网络，如有侵权，请联系作者删除！

QueryContext

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com