org.apache.hadoop.hive.ql.metadata.Table.getBucketCols()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(11.5k)|赞(0)|评价(0)|浏览(123)

本文整理了Java中org.apache.hadoop.hive.ql.metadata.Table.getBucketCols()方法的一些代码示例，展示了Table.getBucketCols()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Table.getBucketCols()方法的具体详情如下：
包路径：org.apache.hadoop.hive.ql.metadata.Table
类名称：Table
方法名：getBucketCols

Table.getBucketCols介绍

暂无

代码示例

代码示例来源：origin: apache/hive

private ArrayList<ExprNodeDesc> getPartitionColsFromBucketCols(String dest, QB qb, Table tab,
                                TableDesc table_desc, Operator input, boolean convert)
  throws SemanticException {
 List<String> tabBucketCols = tab.getBucketCols();
 List<FieldSchema> tabCols = tab.getCols();
 // Partition by the bucketing column
 List<Integer> posns = new ArrayList<Integer>();
 for (String bucketCol : tabBucketCols) {
  int pos = 0;
  for (FieldSchema tabCol : tabCols) {
   if (bucketCol.equals(tabCol.getName())) {
    posns.add(pos);
    break;
   }
   pos++;
  }
 }
 return genConvertCol(dest, qb, tab, table_desc, input, posns, convert);
}

代码示例来源：origin: apache/hive

@Override
public RelDistribution getDistribution() {
 ImmutableList.Builder<Integer> columnPositions = new ImmutableList.Builder<Integer>();
 for (String bucketColumn : this.hiveTblMetadata.getBucketCols()) {
  for (int i=0; i<this.hiveTblMetadata.getSd().getCols().size(); i++) {
   FieldSchema field = this.hiveTblMetadata.getSd().getCols().get(i);
   if (field.getName().equals(bucketColumn)) {
    columnPositions.add(i);
    break;
   }
  }
 }
 return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED,
     columnPositions.build());
}

代码示例来源：origin: apache/drill

private ArrayList<ExprNodeDesc> getPartitionColsFromBucketCols(String dest, QB qb, Table tab,
  TableDesc table_desc, Operator input, boolean convert)
  throws SemanticException {
 List<String> tabBucketCols = tab.getBucketCols();
 List<FieldSchema> tabCols = tab.getCols();
 // Partition by the bucketing column
 List<Integer> posns = new ArrayList<Integer>();
 for (String bucketCol : tabBucketCols) {
  int pos = 0;
  for (FieldSchema tabCol : tabCols) {
   if (bucketCol.equals(tabCol.getName())) {
    posns.add(pos);
    break;
   }
   pos++;
  }
 }
 return genConvertCol(dest, qb, tab, table_desc, input, posns, convert);
}

代码示例来源：origin: apache/hive

private void analyzeAlterTableBucketNum(ASTNode ast, String tblName,
  HashMap<String, String> partSpec) throws SemanticException {
 Table tab = getTable(tblName, true);
 if (tab.getBucketCols() == null || tab.getBucketCols().isEmpty()) {
  throw new SemanticException(ErrorMsg.ALTER_BUCKETNUM_NONBUCKETIZED_TBL.getMsg());
 }
 validateAlterTableType(tab, AlterTableTypes.ALTERBUCKETNUM);
 inputs.add(new ReadEntity(tab));
 int bucketNum = Integer.parseInt(ast.getChild(0).getText());
 AlterTableDesc alterBucketNum = new AlterTableDesc(tblName, partSpec, bucketNum);
 rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
   alterBucketNum)));
}

代码示例来源：origin: apache/drill

private void analyzeAlterTableBucketNum(ASTNode ast, String tblName,
  HashMap<String, String> partSpec) throws SemanticException {
 Table tab = getTable(tblName, true);
 if (tab.getBucketCols() == null || tab.getBucketCols().isEmpty()) {
  throw new SemanticException(ErrorMsg.ALTER_BUCKETNUM_NONBUCKETIZED_TBL.getMsg());
 }
 validateAlterTableType(tab, AlterTableTypes.ALTERBUCKETNUM);
 inputs.add(new ReadEntity(tab));
 int bucketNum = Integer.parseInt(ast.getChild(0).getText());
 AlterTableDesc alterBucketNum = new AlterTableDesc(tblName, partSpec, bucketNum);
 rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
   alterBucketNum), conf));
}

代码示例来源：origin: apache/hive

public Integer splitCount(HiveTableScan scan, RelMetadataQuery mq) {
 Integer splitCount;
 RelOptHiveTable table = (RelOptHiveTable) scan.getTable();
 List<String> bucketCols = table.getHiveTableMD().getBucketCols();
 if (bucketCols != null && !bucketCols.isEmpty()) {
  splitCount = table.getHiveTableMD().getNumBuckets();
 } else {
  splitCount = splitCountRepartition(scan, mq);
  if (splitCount == null) {
   throw new RuntimeException("Could not get split count for table: "
     + scan.getTable().getQualifiedName());
  }
 }
 return splitCount;
}

代码示例来源：origin: apache/drill

public Integer splitCount(HiveTableScan scan, RelMetadataQuery mq) {
 Integer splitCount;
 RelOptHiveTable table = (RelOptHiveTable) scan.getTable();
 List<String> bucketCols = table.getHiveTableMD().getBucketCols();
 if (bucketCols != null && !bucketCols.isEmpty()) {
  splitCount = table.getHiveTableMD().getNumBuckets();
 } else {
  splitCount = splitCountRepartition(scan, mq);
  if (splitCount == null) {
   throw new RuntimeException("Could not get split count for table: "
     + scan.getTable().getQualifiedName());
  }
 }
 return splitCount;
}

代码示例来源：origin: apache/drill

@Override
public RelDistribution getDistribution() {
 ImmutableList.Builder<Integer> columnPositions = new ImmutableList.Builder<Integer>();
 for (String bucketColumn : this.hiveTblMetadata.getBucketCols()) {
  for (int i=0; i<this.hiveTblMetadata.getSd().getCols().size(); i++) {
   FieldSchema field = this.hiveTblMetadata.getSd().getCols().get(i);
   if (field.getName().equals(bucketColumn)) {
    columnPositions.add(i);
    break;
   }
  }
 }
 return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED,
     columnPositions.build());
}

代码示例来源：origin: apache/hive

private boolean checkTable(Table table,
  List<Integer> bucketPositionsDest,
  List<Integer> sortPositionsDest,
  List<Integer> sortOrderDest,
  int numBucketsDest) {
 // The bucketing and sorting positions should exactly match
 int numBuckets = table.getNumBuckets();
 if (numBucketsDest != numBuckets) {
  return false;
 }
 List<Integer> tableBucketPositions =
   getBucketPositions(table.getBucketCols(), table.getCols());
 List<Integer> sortPositions =
   getSortPositions(table.getSortCols(), table.getCols());
 List<Integer> sortOrder =
   getSortOrder(table.getSortCols(), table.getCols());
 return bucketPositionsDest.equals(tableBucketPositions) &&
   sortPositionsDest.equals(sortPositions) &&
   sortOrderDest.equals(sortOrder);
}

代码示例来源：origin: apache/hive

/**
 * Assert that we are not asked to update a bucketing column or partition column.
 * @param colName it's the A in "SET A = B"
 */
protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) throws SemanticException {
 String columnName = normalizeColName(colName.getText());
 // Make sure this isn't one of the partitioning columns, that's not supported.
 for (FieldSchema fschema : targetTable.getPartCols()) {
  if (fschema.getName().equalsIgnoreCase(columnName)) {
   throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg());
  }
 }
 //updating bucket column should move row from one file to another - not supported
 if (targetTable.getBucketCols() != null && targetTable.getBucketCols().contains(columnName)) {
  throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE, columnName);
 }
 boolean foundColumnInTargetTable = false;
 for (FieldSchema col : targetTable.getCols()) {
  if (columnName.equalsIgnoreCase(col.getName())) {
   foundColumnInTargetTable = true;
   break;
  }
 }
 if (!foundColumnInTargetTable) {
  throw new SemanticException(ErrorMsg.INVALID_TARGET_COLUMN_IN_SET_CLAUSE, colName.getText(),
   targetTable.getFullyQualifiedName());
 }
}

代码示例来源：origin: apache/drill

private boolean checkTable(Table table,
  List<Integer> bucketPositionsDest,
  List<Integer> sortPositionsDest,
  List<Integer> sortOrderDest,
  int numBucketsDest) {
 // The bucketing and sorting positions should exactly match
 int numBuckets = table.getNumBuckets();
 if (numBucketsDest != numBuckets) {
  return false;
 }
 List<Integer> tableBucketPositions =
   getBucketPositions(table.getBucketCols(), table.getCols());
 List<Integer> sortPositions =
   getSortPositions(table.getSortCols(), table.getCols());
 List<Integer> sortOrder =
   getSortOrder(table.getSortCols(), table.getCols());
 return bucketPositionsDest.equals(tableBucketPositions) &&
   sortPositionsDest.equals(sortPositions) &&
   sortOrderDest.equals(sortOrder);
}

代码示例来源：origin: apache/hive

List<String> bucketCols = table.getBucketCols();
if (bucketCols != null && !bucketCols.isEmpty()) {

代码示例来源：origin: apache/hive

int numBuckets = -1;
if (isBucketed) {
 bucketColsList.add(table.getBucketCols());
 numBuckets = table.getNumBuckets();
 List<String> sortCols = new ArrayList<String>();

代码示例来源：origin: apache/drill

int numBuckets = -1;
if (isBucketed) {
 bucketColsList.add(table.getBucketCols());
 numBuckets = table.getNumBuckets();
 List<String> sortCols = new ArrayList<String>();

代码示例来源：origin: apache/hive

List<String> bucketCols = table.getBucketCols();
 return matchBucketSortCols(groupByCols, bucketCols, sortCols);
} else {

代码示例来源：origin: apache/drill

/**
 * Assert that we are not asked to update a bucketing column or partition column
 * @param colName it's the A in "SET A = B"
 */
private void checkValidSetClauseTarget(ASTNode colName, Table targetTable) throws SemanticException {
 String columnName = normalizeColName(colName.getText());
 // Make sure this isn't one of the partitioning columns, that's not supported.
 for (FieldSchema fschema : targetTable.getPartCols()) {
  if (fschema.getName().equalsIgnoreCase(columnName)) {
   throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg());
  }
 }
 //updating bucket column should move row from one file to another - not supported
 if(targetTable.getBucketCols() != null && targetTable.getBucketCols().contains(columnName)) {
  throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE,columnName);
 }
 boolean foundColumnInTargetTable = false;
 for(FieldSchema col : targetTable.getCols()) {
  if(columnName.equalsIgnoreCase(col.getName())) {
   foundColumnInTargetTable = true;
   break;
  }
 }
 if(!foundColumnInTargetTable) {
  throw new SemanticException(ErrorMsg.INVALID_TARGET_COLUMN_IN_SET_CLAUSE, colName.getText(),
   getDotName(new String[] {targetTable.getDbName(), targetTable.getTableName()}));
 }
}
private ASTNode findLHSofAssignment(ASTNode assignment) {

代码示例来源：origin: apache/drill

final int nbuckets = tbl.getNumBuckets();
ctxt.setNumBuckets(nbuckets);
ctxt.setBucketCols(tbl.getBucketCols());
ctxt.setSchema(tbl.getFields());
if (tbl.isPartitioned()) {

代码示例来源：origin: apache/hive

private void alterPartitionSpecInMemory(Table tbl,
  Map<String, String> partSpec,
  org.apache.hadoop.hive.metastore.api.Partition tpart,
  boolean inheritTableSpecs,
  String partPath) throws HiveException, InvalidOperationException {
 LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : "
   + partSpec);
 if (inheritTableSpecs) {
  tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat());
  tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat());
  tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib());
  tpart.getSd().getSerdeInfo().setParameters(
    tbl.getTTable().getSd().getSerdeInfo().getParameters());
  tpart.getSd().setBucketCols(tbl.getBucketCols());
  tpart.getSd().setNumBuckets(tbl.getNumBuckets());
  tpart.getSd().setSortCols(tbl.getSortCols());
 }
 if (partPath == null || partPath.trim().equals("")) {
  throw new HiveException("new partition path should not be null or empty.");
 }
 tpart.getSd().setLocation(partPath);
}

代码示例来源：origin: apache/drill

private void alterPartitionSpecInMemory(Table tbl,
  Map<String, String> partSpec,
  org.apache.hadoop.hive.metastore.api.Partition tpart,
  boolean inheritTableSpecs,
  String partPath) throws HiveException, InvalidOperationException {
 LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : "
   + partSpec);
 if (inheritTableSpecs) {
  tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat());
  tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat());
  tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib());
  tpart.getSd().getSerdeInfo().setParameters(
    tbl.getTTable().getSd().getSerdeInfo().getParameters());
  tpart.getSd().setBucketCols(tbl.getBucketCols());
  tpart.getSd().setNumBuckets(tbl.getNumBuckets());
  tpart.getSd().setSortCols(tbl.getSortCols());
 }
 if (partPath == null || partPath.trim().equals("")) {
  throw new HiveException("new partition path should not be null or empty.");
 }
 tpart.getSd().setLocation(partPath);
}

代码示例来源：origin: apache/hive

List<String> bucketCols = tbl.getBucketCols();
bucketCols.add("col1");
try {

内容来源于网络，如有侵权，请联系作者删除！

Table

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Table类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com