本文整理了Java中org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration
类的一些代码示例,展示了AllocationConfiguration
类的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。AllocationConfiguration
类的具体详情如下:
包路径:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration
类名称:AllocationConfiguration
暂无
代码示例来源:origin: apache/hive
allocConf.set(new AllocationConfiguration(conf));
QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy();
if (queuePolicy != null) {
requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueE"));
assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(15, queueConf.getQueueMaxApps("root.queueA"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueB"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueC"));
assertEquals(3, queueConf.getQueueMaxApps("root.queueD"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueE"));
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root"));
assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root.queueA"));
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
queueConf.getMaxResources("root.queueA").getResource());
assertEquals(Resources.createResource(5120, 110),
queueConf.getMaxResources("root.queueB").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueC").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueD").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueE").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueF").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueG").getResource());
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueG.queueH").getResource());
queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(5120, 0),
queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueE"));
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
/**
* Initialize a {@link FSQueue} with queue-specific properties and its
* metrics.
* @param queue the FSQueue needed to be initialized
*/
public void initFSQueue(FSQueue queue){
// Set queue-specific properties.
String name = queue.getName();
queue.setWeights(getQueueWeight(name));
queue.setMinShare(getMinResources(name));
queue.setMaxShare(getMaxResources(name));
queue.setMaxRunningApps(getQueueMaxApps(name));
queue.setMaxAMShare(getQueueMaxAMShare(name));
queue.setMaxChildQueueResource(getMaxChildResources(name));
queue.setMaxContainerAllocation(getQueueMaxContainerAllocation(name));
// Set queue metrics.
queue.getMetrics().setMinShare(queue.getMinShare());
queue.getMetrics().setMaxShare(queue.getMaxShare());
queue.getMetrics().setMaxApps(queue.getMaxRunningApps());
queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
}
}
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
String reservableQueueName = "root.reservable";
String nonreservableQueueName = "root.other";
assertFalse(allocConf.isReservable(nonreservableQueueName));
assertTrue(allocConf.isReservable(reservableQueueName));
Map<FSQueueType, Set<String>> configuredQueues =
allocConf.getConfiguredQueues();
assertTrue("reservable queue is expected be to a parent queue",
configuredQueues.get(FSQueueType.PARENT).contains(reservableQueueName));
assertTrue(allocConf.getMoveOnExpiry(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW,
allocConf.getReservationWindow(reservableQueueName));
assertEquals(100, allocConf.getInstantaneousMaxCapacity
(reservableQueueName),
0.0001);
assertEquals(
"DummyAgentName",
allocConf.getReservationAgent(reservableQueueName));
assertEquals(100, allocConf.getAverageCapacity(reservableQueueName), 0.001);
assertFalse(allocConf.getShowReservationAsQueues(reservableQueueName));
assertEquals("AnyAdmissionPolicy",
allocConf.getReservationAdmissionPolicy(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration
.DEFAULT_RESERVATION_PLANNER_NAME,
allocConf.getReplanner(reservableQueueName));
assertEquals(ReservationSchedulerConfiguration
.DEFAULT_RESERVATION_ENFORCEMENT_WINDOW,
allocConf.getEnforcementWindow(reservableQueueName));
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
QueuePlacementPolicy policy = allocConf.getPlacementPolicy();
List<QueuePlacementRule> rules = policy.getRules();
assertEquals(1, rules.size());
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
.size());
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
.contains("root.queueA"));
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
.contains("root.queueB"));
policy = allocConf.getPlacementPolicy();
rules = policy.getRules();
assertEquals(3, rules.size());
((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
.size());
assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
.contains("root.queueB"));
代码示例来源:origin: com.cloudera.llama/llama
/**
* Assign reservation to a queue based on the placement policy specified
* in the alloc conf
*/
// Visible for testing
String assignToQueue(TLlamaAMReservationRequest request)
throws LlamaException {
// Default means no queue requested
String requestedQueue = (request.isSetQueue()) ? request.getQueue()
: YarnConfiguration.DEFAULT_QUEUE_NAME;
if (requestedQueue == null) {
requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
}
String user = request.getUser();
String queue;
try {
queue = allocConf.get().getPlacementPolicy()
.assignAppToQueue(requestedQueue, user);
} catch (IOException ex) {
throw new LlamaException(ex, ErrorCode.INTERNAL_ERROR);
}
if (queue == null) {
throw new LlamaException(
ErrorCode.RESERVATION_USER_TO_QUEUE_MAPPING_NOT_FOUND, user, requestedQueue);
}
LOG.debug("Reservation from user " + user + " with requested queue " +
requestedQueue + " resolved to queue " + queue);
return queue;
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
/**
* Update the min/fair share preemption timeouts and threshold for this queue.
*/
public void updatePreemptionVariables() {
// For min share timeout
minSharePreemptionTimeout = scheduler.getAllocationConfiguration()
.getMinSharePreemptionTimeout(getName());
if (minSharePreemptionTimeout == -1 && parent != null) {
minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout();
}
// For fair share timeout
fairSharePreemptionTimeout = scheduler.getAllocationConfiguration()
.getFairSharePreemptionTimeout(getName());
if (fairSharePreemptionTimeout == -1 && parent != null) {
fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout();
}
// For fair share preemption threshold
fairSharePreemptionThreshold = scheduler.getAllocationConfiguration()
.getFairSharePreemptionThreshold(getName());
if (fairSharePreemptionThreshold < 0 && parent != null) {
fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold();
}
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
maxApps = allocConf.getQueueMaxApps(queueName);
if (allocConf.isReservable(queueName) &&
!allocConf.getShowReservationAsQueues(queueName)) {
return;
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
.getMinSharePreemptionTimeout(getName());
if (minSharePreemptionTimeout == -1 && parent != null) {
minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout();
.getFairSharePreemptionTimeout(getName());
if (fairSharePreemptionTimeout == -1 && parent != null) {
fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout();
.getFairSharePreemptionThreshold(getName());
if (fairSharePreemptionThreshold < 0 && parent != null) {
fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold();
} else {
preemptable = scheduler.getAllocationConfiguration()
.isPreemptable(getName());
代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager
/**
* Checks whether making the application runnable would exceed any
* maxRunningApps limits.
*/
public boolean canAppBeRunnable(FSQueue queue, String user) {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
Integer userNumRunnable = usersNumRunnableApps.get(user);
if (userNumRunnable == null) {
userNumRunnable = 0;
}
if (userNumRunnable >= allocConf.getUserMaxApps(user)) {
return false;
}
// Check queue and all parent queues
while (queue != null) {
int queueMaxApps = allocConf.getQueueMaxApps(queue.getName());
if (queue.getNumRunnableApps() >= queueMaxApps) {
return false;
}
queue = queue.getParent();
}
return true;
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
@Override
public Resource getMaxShare() {
return scheduler.getAllocationConfiguration().getMaxResources(getName());
}
代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager
public void updateAllocationConfiguration(AllocationConfiguration queueConf) {
for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) {
if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) {
getLeafQueue(name, true);
for (String name : queueConf.getConfiguredQueues().get(
FSQueueType.PARENT)) {
if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName());
policy.initialize(scheduler.getClusterResource());
queue.setPolicy(policy);
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
@Test
public void testUpdateDemand() {
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
resourceManager = new MockRM(conf);
resourceManager.start();
scheduler = (FairScheduler) resourceManager.getResourceScheduler();
scheduler.allocConf = mock(AllocationConfiguration.class);
String queueName = "root.queue1";
when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
schedulable.addAppSchedulable(app);
schedulable.addAppSchedulable(app);
schedulable.updateDemand();
assertTrue("Demand is greater than max allowed ",
Resources.equals(schedulable.getDemand(), maxResource));
}
代码示例来源:origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager
private Set<String> getRemovedStaticQueues(
AllocationConfiguration queueInfo) {
if (queueInfo == null || allocConf == null) {
return Collections.emptySet();
}
Set<String> removedStaticQueues = new HashSet<>();
for (Set<String> queues : allocConf.getConfiguredQueues().values()) {
removedStaticQueues.addAll(queues);
}
for (Set<String> queues : queueInfo.getConfiguredQueues().values()) {
removedStaticQueues.removeAll(queues);
}
return removedStaticQueues;
}
代码示例来源:origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager
/**
* Traverses the queue hierarchy under the given queue to gather all lists
* of non-runnable applications.
*/
private void gatherPossiblyRunnableAppLists(FSQueue queue,
List<List<FSAppAttempt>> appLists) {
if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration()
.getQueueMaxApps(queue.getName())) {
if (queue instanceof FSLeafQueue) {
appLists.add(
((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
} else {
for (FSQueue child : queue.getChildQueues()) {
gatherPossiblyRunnableAppLists(child, appLists);
}
}
}
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
@Override
public Resource getMinShare() {
return scheduler.getAllocationConfiguration().getMinResources(getName());
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
/**
* Check whether this queue can run this application master under the
* maxAMShare limit
*
* @param amResource
* @return true if this queue can run
*/
public boolean canRunAppAM(Resource amResource) {
float maxAMShare =
scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
return true;
}
Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare);
Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
return !policy
.checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource);
}
代码示例来源:origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager
assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
queueConf.getMinResources("root.queueA"));
assertEquals(Resources.createResource(2048, 0),
queueConf.getMinResources("root.queueB"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueC"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueE"));
assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(15, queueConf.getQueueMaxApps("root.queueA"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueB"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueC"));
assertEquals(3, queueConf.getQueueMaxApps("root.queueD"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueE"));
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
assertEquals(" ", queueConf.getQueueAcl("root.queueA",
QueueACL.ADMINISTER_QUEUE).getAclString());
assertEquals(" ", queueConf.getQueueAcl("root.queueA",
QueueACL.SUBMIT_APPLICATIONS).getAclString());
内容来源于网络,如有侵权,请联系作者删除!