org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration类的使用及代码示例

x33g5p2x  于2022-01-17 转载在 其他  
字(16.1k)|赞(0)|评价(0)|浏览(194)

本文整理了Java中org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration类的一些代码示例,展示了AllocationConfiguration类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。AllocationConfiguration类的具体详情如下:
包路径:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration
类名称:AllocationConfiguration

AllocationConfiguration介绍

暂无

代码示例

代码示例来源:origin: apache/hive

allocConf.set(new AllocationConfiguration(conf));
QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy();
if (queuePolicy != null) {
 requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
  queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
  queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueE"));
assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(15, queueConf.getQueueMaxApps("root.queueA"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueB"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueC"));
assertEquals(3, queueConf.getQueueMaxApps("root.queueD"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueE"));
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root"));
assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." +
  YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root.queueA"));

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
  queueConf.getMaxResources("root.queueA").getResource());
assertEquals(Resources.createResource(5120, 110),
  queueConf.getMaxResources("root.queueB").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueC").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueD").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueE").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueF").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueG").getResource());
assertEquals(Resources.createResource(4096, 100),
  queueConf.getMaxResources("root.queueG.queueH").getResource());
  queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
  queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(5120, 0),
  queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueE"));

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

/**
  * Initialize a {@link FSQueue} with queue-specific properties and its
  * metrics.
  * @param queue the FSQueue needed to be initialized
  */
 public void initFSQueue(FSQueue queue){
  // Set queue-specific properties.
  String name = queue.getName();
  queue.setWeights(getQueueWeight(name));
  queue.setMinShare(getMinResources(name));
  queue.setMaxShare(getMaxResources(name));
  queue.setMaxRunningApps(getQueueMaxApps(name));
  queue.setMaxAMShare(getQueueMaxAMShare(name));
  queue.setMaxChildQueueResource(getMaxChildResources(name));
  queue.setMaxContainerAllocation(getQueueMaxContainerAllocation(name));

  // Set queue metrics.
  queue.getMetrics().setMinShare(queue.getMinShare());
  queue.getMetrics().setMaxShare(queue.getMaxShare());
  queue.getMetrics().setMaxApps(queue.getMaxRunningApps());
  queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
 }
}

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

String reservableQueueName = "root.reservable";
String nonreservableQueueName = "root.other";
assertFalse(allocConf.isReservable(nonreservableQueueName));
assertTrue(allocConf.isReservable(reservableQueueName));
Map<FSQueueType, Set<String>> configuredQueues =
  allocConf.getConfiguredQueues();
assertTrue("reservable queue is expected be to a parent queue",
  configuredQueues.get(FSQueueType.PARENT).contains(reservableQueueName));
assertTrue(allocConf.getMoveOnExpiry(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW,
  allocConf.getReservationWindow(reservableQueueName));
assertEquals(100, allocConf.getInstantaneousMaxCapacity
    (reservableQueueName),
  0.0001);
assertEquals(
  "DummyAgentName",
  allocConf.getReservationAgent(reservableQueueName));
assertEquals(100, allocConf.getAverageCapacity(reservableQueueName), 0.001);
assertFalse(allocConf.getShowReservationAsQueues(reservableQueueName));
assertEquals("AnyAdmissionPolicy",
  allocConf.getReservationAdmissionPolicy(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration
  .DEFAULT_RESERVATION_PLANNER_NAME,
  allocConf.getReplanner(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration
  .DEFAULT_RESERVATION_ENFORCEMENT_WINDOW,
  allocConf.getEnforcementWindow(reservableQueueName));

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
 AllocationConfiguration allocConf = new AllocationConfiguration(conf);
 allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
 queueMgr.updateAllocationConfiguration(allocConf);
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

QueuePlacementPolicy policy = allocConf.getPlacementPolicy();
List<QueuePlacementRule> rules = policy.getRules();
assertEquals(1, rules.size());
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
  .size());
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
  .contains("root.queueA"));
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
  .contains("root.queueB"));
policy = allocConf.getPlacementPolicy();
rules = policy.getRules();
assertEquals(3, rules.size());
  ((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
  .size());
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
  .contains("root.queueB"));

代码示例来源:origin: com.cloudera.llama/llama

/**
 * Assign reservation to a queue based on the placement policy specified
 * in the alloc conf
 */
// Visible for testing
String assignToQueue(TLlamaAMReservationRequest request)
  throws LlamaException {
 // Default means no queue requested
 String requestedQueue = (request.isSetQueue()) ? request.getQueue()
   : YarnConfiguration.DEFAULT_QUEUE_NAME;
 if (requestedQueue == null) {
  requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
 }
 String user = request.getUser();
 String queue;
 try {
  queue = allocConf.get().getPlacementPolicy()
    .assignAppToQueue(requestedQueue, user);
 } catch (IOException ex) {
  throw new LlamaException(ex, ErrorCode.INTERNAL_ERROR);
 }
 if (queue == null) {
  throw new LlamaException(
    ErrorCode.RESERVATION_USER_TO_QUEUE_MAPPING_NOT_FOUND, user, requestedQueue);
 }
 LOG.debug("Reservation from user " + user + " with requested queue " +
   requestedQueue + " resolved to queue " + queue);
 
 return queue;
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Update the min/fair share preemption timeouts and threshold for this queue.
 */
public void updatePreemptionVariables() {
 // For min share timeout
 minSharePreemptionTimeout = scheduler.getAllocationConfiguration()
   .getMinSharePreemptionTimeout(getName());
 if (minSharePreemptionTimeout == -1 && parent != null) {
  minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout();
 }
 // For fair share timeout
 fairSharePreemptionTimeout = scheduler.getAllocationConfiguration()
   .getFairSharePreemptionTimeout(getName());
 if (fairSharePreemptionTimeout == -1 && parent != null) {
  fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout();
 }
 // For fair share preemption threshold
 fairSharePreemptionThreshold = scheduler.getAllocationConfiguration()
   .getFairSharePreemptionThreshold(getName());
 if (fairSharePreemptionThreshold < 0 && parent != null) {
  fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold();
 }
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
maxApps = allocConf.getQueueMaxApps(queueName);
if (allocConf.isReservable(queueName) &&
  !allocConf.getShowReservationAsQueues(queueName)) {
 return;

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

.getMinSharePreemptionTimeout(getName());
if (minSharePreemptionTimeout == -1 && parent != null) {
 minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout();
  .getFairSharePreemptionTimeout(getName());
if (fairSharePreemptionTimeout == -1 && parent != null) {
 fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout();
  .getFairSharePreemptionThreshold(getName());
if (fairSharePreemptionThreshold < 0 && parent != null) {
 fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold();
} else {
 preemptable = scheduler.getAllocationConfiguration()
   .isPreemptable(getName());

代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

/**
 * Checks whether making the application runnable would exceed any
 * maxRunningApps limits.
 */
public boolean canAppBeRunnable(FSQueue queue, String user) {
 AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
 Integer userNumRunnable = usersNumRunnableApps.get(user);
 if (userNumRunnable == null) {
  userNumRunnable = 0;
 }
 if (userNumRunnable >= allocConf.getUserMaxApps(user)) {
  return false;
 }
 // Check queue and all parent queues
 while (queue != null) {
  int queueMaxApps = allocConf.getQueueMaxApps(queue.getName());
  if (queue.getNumRunnableApps() >= queueMaxApps) {
   return false;
  }
  queue = queue.getParent();
 }
 return true;
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

@Override
public Resource getMaxShare() {
 return scheduler.getAllocationConfiguration().getMaxResources(getName());
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

public void updateAllocationConfiguration(AllocationConfiguration queueConf) {
 for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) {
  if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) {
   getLeafQueue(name, true);
 for (String name : queueConf.getConfiguredQueues().get(
   FSQueueType.PARENT)) {
  if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
   SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName());
   policy.initialize(scheduler.getClusterResource());
   queue.setPolicy(policy);

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

@Test
public void testUpdateDemand() {
 conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
 resourceManager = new MockRM(conf);
 resourceManager.start();
 scheduler = (FairScheduler) resourceManager.getResourceScheduler();
 scheduler.allocConf = mock(AllocationConfiguration.class);
 String queueName = "root.queue1";
 when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
 when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
 FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
 FSAppAttempt app = mock(FSAppAttempt.class);
 Mockito.when(app.getDemand()).thenReturn(maxResource);
 schedulable.addAppSchedulable(app);
 schedulable.addAppSchedulable(app);
 schedulable.updateDemand();
 assertTrue("Demand is greater than max allowed ",
   Resources.equals(schedulable.getDemand(), maxResource));
}

代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

private Set<String> getRemovedStaticQueues(
  AllocationConfiguration queueInfo) {
 if (queueInfo == null || allocConf == null) {
  return Collections.emptySet();
 }
 Set<String> removedStaticQueues = new HashSet<>();
 for (Set<String> queues : allocConf.getConfiguredQueues().values()) {
  removedStaticQueues.addAll(queues);
 }
 for (Set<String> queues : queueInfo.getConfiguredQueues().values()) {
  removedStaticQueues.removeAll(queues);
 }
 return removedStaticQueues;
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

/**
 * Traverses the queue hierarchy under the given queue to gather all lists
 * of non-runnable applications.
 */
private void gatherPossiblyRunnableAppLists(FSQueue queue,
  List<List<FSAppAttempt>> appLists) {
 if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration()
   .getQueueMaxApps(queue.getName())) {
  if (queue instanceof FSLeafQueue) {
   appLists.add(
     ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
  } else {
   for (FSQueue child : queue.getChildQueues()) {
    gatherPossiblyRunnableAppLists(child, appLists);
   }
  }
 }
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

@Override
public Resource getMinShare() {
 return scheduler.getAllocationConfiguration().getMinResources(getName());
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Check whether this queue can run this application master under the
 * maxAMShare limit
 *
 * @param amResource
 * @return true if this queue can run
 */
public boolean canRunAppAM(Resource amResource) {
 float maxAMShare =
   scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
 if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
  return true;
 }
 Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare);
 Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
 return !policy
   .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource);
}

代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
  queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
  queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
  queueConf.getMinResources("root.queueE"));
assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(15, queueConf.getQueueMaxApps("root.queueA"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueB"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueC"));
assertEquals(3, queueConf.getQueueMaxApps("root.queueD"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueE"));
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
assertEquals(" ", queueConf.getQueueAcl("root.queueA",
  QueueACL.ADMINISTER_QUEUE).getAclString());
assertEquals(" ", queueConf.getQueueAcl("root.queueA",
  QueueACL.SUBMIT_APPLICATIONS).getAclString());

相关文章

微信公众号

最新文章

更多