org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy类的使用及代码示例

x33g5p2x  于2022-01-28 转载在 其他  
字(14.3k)|赞(0)|评价(0)|浏览(274)

本文整理了Java中org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy类的一些代码示例，展示了QueuePlacementPolicy类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。QueuePlacementPolicy类的具体详情如下：
包路径：org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy
类名称：QueuePlacementPolicy

QueuePlacementPolicy介绍

暂无

代码示例

代码示例来源：origin: apache/hive

requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
if (StringUtils.isNotBlank(requestedQueue)) {
 LOG.debug("Setting queue name to " + requestedQueue + " for user "

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

private QueuePlacementPolicy getQueuePlacementPolicy(
  AllocationFileParser allocationFileParser,
  QueueProperties queueProperties, Configuration conf)
  throws AllocationConfigurationException {
 if (allocationFileParser.getQueuePlacementPolicy().isPresent()) {
  return QueuePlacementPolicy.fromXml(
    allocationFileParser.getQueuePlacementPolicy().get(),
    queueProperties.getConfiguredQueues(), conf);
 } else {
  return QueuePlacementPolicy.fromConfiguration(conf,
    queueProperties.getConfiguredQueues());
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Builds a QueuePlacementPolicy from an xml element.
 */
public static QueuePlacementPolicy fromXml(Element el,
  Map<FSQueueType, Set<String>> configuredQueues, Configuration conf)
  throws AllocationConfigurationException {
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 NodeList elements = el.getChildNodes();
 for (int i = 0; i < elements.getLength(); i++) {
  Node node = elements.item(i);
  if (node instanceof Element) {
   QueuePlacementRule rule = createAndInitializeRule(node);
   rules.add(rule);
  }
 }
 return new QueuePlacementPolicy(rules, configuredQueues, conf);
}

代码示例来源：origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

/**
 * Build a simple queue placement policy from the allow-undeclared-pools and
 * user-as-default-queue configuration options.
 */
public static QueuePlacementPolicy fromConfiguration(Configuration conf,
  Map<FSQueueType, Set<String>> configuredQueues) {
 boolean create = conf.getBoolean(
   FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
   FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS);
 boolean userAsDefaultQueue = conf.getBoolean(
   FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
   FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE);
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 rules.add(new QueuePlacementRule.Specified().initialize(create, null));
 if (userAsDefaultQueue) {
  rules.add(new QueuePlacementRule.User().initialize(create, null));
 }
 if (!userAsDefaultQueue || !create) {
  rules.add(new QueuePlacementRule.Default().initialize(true, null));
 }
 try {
  return new QueuePlacementPolicy(rules, configuredQueues, conf);
 } catch (AllocationConfigurationException ex) {
  throw new RuntimeException("Should never hit exception when loading" +
      "placement policy from conf", ex);
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

private QueuePlacementPolicy parse(String str) throws Exception {
  // Read and parse the allocations file.
  DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
    .newInstance();
  docBuilderFactory.setIgnoringComments(true);
  DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
  Document doc = builder.parse(IOUtils.toInputStream(str));
  Element root = doc.getDocumentElement();
  return QueuePlacementPolicy.fromXml(root, configuredQueues, conf);
 }
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

public AllocationConfiguration(Configuration conf) {
 minQueueResources = new HashMap<String, Resource>();
 maxQueueResources = new HashMap<String, Resource>();
 queueWeights = new HashMap<String, ResourceWeights>();
 queueMaxApps = new HashMap<String, Integer>();
 userMaxApps = new HashMap<String, Integer>();
 queueMaxAMShares = new HashMap<String, Float>();
 userMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxAMShareDefault = 0.5f;
 queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
 minSharePreemptionTimeouts = new HashMap<String, Long>();
 fairSharePreemptionTimeouts = new HashMap<String, Long>();
 fairSharePreemptionThresholds = new HashMap<String, Float>();
 schedulingPolicies = new HashMap<String, SchedulingPolicy>();
 defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
 reservableQueues = new HashSet<>();
 configuredQueues = new HashMap<FSQueueType, Set<String>>();
 for (FSQueueType queueType : FSQueueType.values()) {
  configuredQueues.put(queueType, new HashSet<String>());
 }
 placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
   configuredQueues);
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

if ("rule".equals(element.getTagName())) {
 QueuePlacementRule rule = QueuePlacementPolicy
   .createAndInitializeRule(node);
 if (rule == null) {
  throw new AllocationConfigurationException(

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

@Test
public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured()
  throws IOException {
 // This test verifies if default rule in queue placement policy
 // initializes properly when policy is not configured and
 // undeclared pools is not allowed.
 conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
 conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
 // Create an alloc file with no queue placement policy
 PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
 out.println("<?xml version=\"1.0\"?>");
 out.println("<allocations>");
 out.println("</allocations>");
 out.close();
 scheduler.init(conf);
 scheduler.start();
 scheduler.reinitialize(conf, resourceManager.getRMContext());
 List<QueuePlacementRule> rules =
   scheduler.allocConf.placementPolicy.getRules();
 for (QueuePlacementRule rule : rules) {
  if (rule instanceof Default) {
   Default defaultRule = (Default) rule;
   assertNotNull(defaultRule.defaultQueueName);
  }
 }
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement,
   configuredQueues, conf);
} else {
 newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
   configuredQueues);

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Builds a QueuePlacementPolicy from an xml element.
 */
public static QueuePlacementPolicy fromXml(Element el,
  Map<FSQueueType, Set<String>> configuredQueues, Configuration conf)
  throws AllocationConfigurationException {
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 NodeList elements = el.getChildNodes();
 for (int i = 0; i < elements.getLength(); i++) {
  Node node = elements.item(i);
  if (node instanceof Element) {
   QueuePlacementRule rule = createAndInitializeRule(node);
   rules.add(rule);
  }
 }
 return new QueuePlacementPolicy(rules, configuredQueues, conf);
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Build a simple queue placement policy from the allow-undeclared-pools and
 * user-as-default-queue configuration options.
 */
public static QueuePlacementPolicy fromConfiguration(Configuration conf,
  Map<FSQueueType, Set<String>> configuredQueues) {
 boolean create = conf.getBoolean(
   FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
   FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS);
 boolean userAsDefaultQueue = conf.getBoolean(
   FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
   FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE);
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 rules.add(new QueuePlacementRule.Specified().initialize(create, null));
 if (userAsDefaultQueue) {
  rules.add(new QueuePlacementRule.User().initialize(create, null));
 }
 if (!userAsDefaultQueue || !create) {
  rules.add(new QueuePlacementRule.Default().initialize(true, null));
 }
 try {
  return new QueuePlacementPolicy(rules, configuredQueues, conf);
 } catch (AllocationConfigurationException ex) {
  throw new RuntimeException("Should never hit exception when loading" +
      "placement policy from conf", ex);
 }
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

private QueuePlacementPolicy parse(String str) throws Exception {
  // Read and parse the allocations file.
  DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
    .newInstance();
  docBuilderFactory.setIgnoringComments(true);
  DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
  Document doc = builder.parse(IOUtils.toInputStream(str));
  Element root = doc.getDocumentElement();
  return QueuePlacementPolicy.fromXml(root, configuredQueues, conf);
 }
}

代码示例来源：origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

public AllocationConfiguration(Configuration conf) {
 minQueueResources = new HashMap<String, Resource>();
 maxQueueResources = new HashMap<String, Resource>();
 queueWeights = new HashMap<String, ResourceWeights>();
 queueMaxApps = new HashMap<String, Integer>();
 userMaxApps = new HashMap<String, Integer>();
 queueMaxAMShares = new HashMap<String, Float>();
 userMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxAMShareDefault = 0.5f;
 queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
 minSharePreemptionTimeouts = new HashMap<String, Long>();
 fairSharePreemptionTimeouts = new HashMap<String, Long>();
 fairSharePreemptionThresholds = new HashMap<String, Float>();
 schedulingPolicies = new HashMap<String, SchedulingPolicy>();
 defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
 reservableQueues = new HashSet<>();
 configuredQueues = new HashMap<FSQueueType, Set<String>>();
 for (FSQueueType queueType : FSQueueType.values()) {
  configuredQueues.put(queueType, new HashSet<String>());
 }
 placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
   configuredQueues);
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

if ("rule".equals(element.getTagName())) {
 QueuePlacementRule rule = QueuePlacementPolicy
   .createAndInitializeRule(node);
 if (rule == null) {
  throw new AllocationConfigurationException(

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

@Test
public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured()
  throws IOException {
 // This test verifies if default rule in queue placement policy
 // initializes properly when policy is not configured and
 // undeclared pools is not allowed.
 conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
 conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
 // Create an alloc file with no queue placement policy
 PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
 out.println("<?xml version=\"1.0\"?>");
 out.println("<allocations>");
 out.println("</allocations>");
 out.close();
 scheduler.init(conf);
 scheduler.start();
 scheduler.reinitialize(conf, resourceManager.getRMContext());
 List<QueuePlacementRule> rules = scheduler.allocConf.placementPolicy
   .getRules();
 for (QueuePlacementRule rule : rules) {
  if (rule instanceof Default) {
   Default defaultRule = (Default) rule;
   assertNotNull(defaultRule.defaultQueueName);
  }
 }
}

代码示例来源：origin: com.github.hyukjinkwon.shims/hive-shims-scheduler

requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
if (StringUtils.isNotBlank(requestedQueue)) {
 LOG.debug("Setting queue name to " + requestedQueue + " for user "

代码示例来源：origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement,
   configuredQueues, conf);
} else {
 newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
   configuredQueues);

代码示例来源：origin: com.github.jiayuhan-it/hadoop-yarn-server-resourcemanager

/**
 * Builds a QueuePlacementPolicy from an xml element.
 */
public static QueuePlacementPolicy fromXml(Element el,
  Map<FSQueueType, Set<String>> configuredQueues, Configuration conf)
  throws AllocationConfigurationException {
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 NodeList elements = el.getChildNodes();
 for (int i = 0; i < elements.getLength(); i++) {
  Node node = elements.item(i);
  if (node instanceof Element) {
   QueuePlacementRule rule = createAndInitializeRule(node);
   rules.add(rule);
  }
 }
 return new QueuePlacementPolicy(rules, configuredQueues, conf);
}

代码示例来源：origin: ch.cern.hadoop/hadoop-yarn-server-resourcemanager

/**
 * Build a simple queue placement policy from the allow-undeclared-pools and
 * user-as-default-queue configuration options.
 */
public static QueuePlacementPolicy fromConfiguration(Configuration conf,
  Map<FSQueueType, Set<String>> configuredQueues) {
 boolean create = conf.getBoolean(
   FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
   FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS);
 boolean userAsDefaultQueue = conf.getBoolean(
   FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
   FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE);
 List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
 rules.add(new QueuePlacementRule.Specified().initialize(create, null));
 if (userAsDefaultQueue) {
  rules.add(new QueuePlacementRule.User().initialize(create, null));
 }
 if (!userAsDefaultQueue || !create) {
  rules.add(new QueuePlacementRule.Default().initialize(true, null));
 }
 try {
  return new QueuePlacementPolicy(rules, configuredQueues, conf);
 } catch (AllocationConfigurationException ex) {
  throw new RuntimeException("Should never hit exception when loading" +
      "placement policy from conf", ex);
 }
}

代码示例来源：origin: org.apache.hadoop/hadoop-yarn-server-resourcemanager

public AllocationConfiguration(Configuration conf) {
 minQueueResources = new HashMap<>();
 maxChildQueueResources = new HashMap<>();
 maxQueueResources = new HashMap<>();
 queueWeights = new HashMap<>();
 queueMaxApps = new HashMap<>();
 userMaxApps = new HashMap<>();
 queueMaxAMShares = new HashMap<>();
 userMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxAppsDefault = Integer.MAX_VALUE;
 queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded());
 queueMaxAMShareDefault = 0.5f;
 queueAcls = new HashMap<>();
 resAcls = new HashMap<>();
 minSharePreemptionTimeouts = new HashMap<>();
 fairSharePreemptionTimeouts = new HashMap<>();
 fairSharePreemptionThresholds = new HashMap<>();
 schedulingPolicies = new HashMap<>();
 defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
 reservableQueues = new HashSet<>();
 configuredQueues = new HashMap<>();
 for (FSQueueType queueType : FSQueueType.values()) {
  configuredQueues.put(queueType, new HashSet<>());
 }
 placementPolicy =
   QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
 nonPreemptableQueues = new HashSet<>();
 queueMaxContainerAllocationMap = new HashMap<>();
}

内容来源于网络，如有侵权，请联系作者删除！

QueuePlacementPolicy

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com