org.apache.hadoop.hive.metastore.api.Partition.setParameters()方法的使用及代码示例

x33g5p2x  于2022-01-26 转载在 其他  
字(12.8k)|赞(0)|评价(0)|浏览(133)

本文整理了Java中org.apache.hadoop.hive.metastore.api.Partition.setParameters()方法的一些代码示例，展示了Partition.setParameters()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Partition.setParameters()方法的具体详情如下：
包路径：org.apache.hadoop.hive.metastore.api.Partition
类名称：Partition
方法名：setParameters

Partition.setParameters介绍

暂无

代码示例

代码示例来源：origin: apache/hive

@Override public void setParameters(Map<String, String> parameters) { partition.setParameters(parameters); }
@Override public void putToParameters(String key, String value) { partition.putToParameters(key, value);}

代码示例来源：origin: apache/hive

@Override
public void setParameters(Map<String, String> parameters) {
 partitionList.get(index).setParameters(parameters);
}

代码示例来源：origin: apache/hive

static void setPartitionParameters(String PARTITION_PARAMS, boolean convertMapNullsToEmptyStrings,
  PersistenceManager pm, String partIds, TreeMap<Long, Partition> partitions)
  throws MetaException {
 String queryText;
 queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + PARTITION_PARAMS + ""
   + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null"
   + " order by \"PART_ID\" asc";
 loopJoinOrderedResult(pm, partitions, queryText, 0, new ApplyFunc<Partition>() {
  @Override
  public void apply(Partition t, Object[] fields) {
   t.putToParameters((String)fields[1], extractSqlClob(fields[2]));
  }});
 // Perform conversion of null map values
 for (Partition t : partitions.values()) {
  t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings));
 }
}

代码示例来源：origin: prestodb/presto

@Override
public synchronized void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update)
{
  PartitionStatistics currentStatistics = requireNonNull(
      getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null");
  PartitionStatistics updatedStatistics = update.apply(currentStatistics);
  List<Partition> partitions = getPartitionsByNames(databaseName, tableName, ImmutableList.of(partitionName));
  if (partitions.size() != 1) {
    throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName);
  }
  Partition originalPartition = getOnlyElement(partitions);
  Partition modifiedPartition = originalPartition.deepCopy();
  HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics();
  modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics));
  alterPartitionWithoutStatistics(databaseName, tableName, modifiedPartition);
  Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream()
      .collect(toImmutableMap(FieldSchema::getName, schema -> HiveType.valueOf(schema.getType())));
  setPartitionColumnStatistics(databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount());
  Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet());
  removedStatistics.forEach(column -> deletePartitionColumnStatistics(databaseName, tableName, partitionName, column));
}

代码示例来源：origin: prestodb/presto

public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(PartitionWithStatistics partitionWithStatistics)
{
  org.apache.hadoop.hive.metastore.api.Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition());
  partition.setParameters(updateStatisticsParameters(partition.getParameters(), partitionWithStatistics.getStatistics().getBasicStatistics()));
  return partition;
}

代码示例来源：origin: apache/hive

@Test
public void isFastStatsSameMatching() {
 Partition oldPartition = new Partition();
 Partition newPartition = new Partition();
 Map<String, String> stats = new HashMap<>();
 Map<String, String> oldParams = new HashMap<>();
 Map<String, String> newParams = new HashMap<>();
 long testVal = 1;
 for (String key : FAST_STATS) {
  oldParams.put(key, String.valueOf(testVal));
  newParams.put(key, String.valueOf(testVal));
 }
 oldPartition.setParameters(oldParams);
 newPartition.setParameters(newParams);
 assertTrue(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition));
}

代码示例来源：origin: apache/hive

@Test
public void isFastStatsSameNullStatsInNew() {
 Partition oldPartition = new Partition();
 Partition newPartition = new Partition();
 Map<String, String> oldParams = new HashMap<>();
 Map<String, String> newParams = new HashMap<>();
 long testVal = 1;
 for (String key : FAST_STATS) {
  oldParams.put(key, String.valueOf(testVal));
  newParams.put(key, null);
 }
 oldPartition.setParameters(oldParams);
 newPartition.setParameters(newParams);
 assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition));
}

代码示例来源：origin: apache/hive

@Test
public void isFastStatsSameDifferent() {
 Partition oldPartition = new Partition();
 Partition newPartition = new Partition();
 Map<String, String> stats = new HashMap<>();
 Map<String, String> oldParams = new HashMap<>();
 Map<String, String> newParams = new HashMap<>();
 long testVal = 1;
 for (String key : FAST_STATS) {
  oldParams.put(key, String.valueOf(testVal));
  newParams.put(key, String.valueOf(++testVal));
 }
 oldPartition.setParameters(oldParams);
 newPartition.setParameters(newParams);
 assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition));
}

代码示例来源：origin: prestodb/presto

@Override
public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics)
{
  Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition());
  if (partition.getParameters() == null) {
    partition.setParameters(ImmutableMap.of());
  }
  PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName());
  partitions.put(partitionKey, partition);
  partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics());
}

代码示例来源：origin: prestodb/presto

@Override
public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics)
{
  for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) {
    Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition());
    if (partition.getParameters() == null) {
      partition.setParameters(ImmutableMap.of());
    }
    PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName());
    partitions.put(partitionKey, partition);
    partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics());
  }
}

代码示例来源：origin: apache/hive

@Test
public void isFastStatsSameWithNoMatchingStats() {
 Partition oldPartition = new Partition();
 Map<String, String> stats = new HashMap<>();
 oldPartition.setParameters(stats);
 assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, null));
 stats.put("someKeyThatIsNotInFastStats","value");
 oldPartition.setParameters(stats);
 assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, null));
}

代码示例来源：origin: apache/incubator-gobblin

@Test
public void testIsOlderThanLookbackForDistcpGenerationTime() throws Exception {
 long currentTime = System.currentTimeMillis();
 // Default lookback time is 3 days
 long partitionCreateTime = new DateTime(currentTime).minusDays(2).getMillis();
 Map<String, String> parameters = Maps.newHashMap();
 parameters.put(HiveSource.DISTCP_REGISTRATION_GENERATION_TIME_KEY, partitionCreateTime + "");
 org.apache.hadoop.hive.ql.metadata.Partition partition = this.hiveMetastoreTestUtils.createDummyPartition(0);
 partition.getTPartition().setParameters(parameters);
 SourceState testState = getTestState("testDb6");
 HiveSource source = new HiveSource();
 source.initialize(testState);
 boolean isOlderThanLookback = source.isOlderThanLookback(partition);
 Assert.assertEquals(isOlderThanLookback, false, "Should create workunits newer than lookback");
}

代码示例来源：origin: apache/hive

@Test
public void isFastStatsSameMatchingButOnlyOneStat() {
 Partition oldPartition = new Partition();
 Partition newPartition = new Partition();
 Map<String, String> randomParams = new HashMap<String, String>();
 randomParams.put("randomParam1", "randomVal1");
 newPartition.setParameters(randomParams);
 assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition));
}

代码示例来源：origin: apache/hive

Partition toHivePartition() throws HCatException {
 Partition hivePtn = new Partition();
 hivePtn.setDbName(dbName);
 hivePtn.setTableName(tableName);
 hivePtn.setValues(values);
 hivePtn.setParameters(parameters);
 if (sd.getLocation() == null) {
  LOG.warn("Partition location is not set! Attempting to construct default partition location.");
  try {
   String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values);
   sd.setLocation(new Path(hcatTable.getSd().getLocation(), partName).toString());
  }
  catch(MetaException exception) {
   throw new HCatException("Could not construct default partition-path for "
     + hcatTable.getDbName() + "." + hcatTable.getTableName() + "[" + values + "]");
  }
 }
 hivePtn.setSd(sd);
 hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000));
 hivePtn.setLastAccessTimeIsSet(false);
 return hivePtn;
}

代码示例来源：origin: prestodb/presto

public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition)
{
  org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition();
  result.setDbName(partition.getDatabaseName());
  result.setTableName(partition.getTableName());
  result.setValues(partition.getValues());
  result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage()));
  result.setParameters(partition.getParameters());
  return result;
}

代码示例来源：origin: apache/hive

protected Partition newPartition(Table t, String value, List<Order> sortCols) throws Exception {
 Partition part = new Partition();
 part.addToValues(value);
 part.setDbName(t.getDbName());
 part.setTableName(t.getTableName());
 part.setSd(newStorageDescriptor(getLocation(t.getTableName(), value), sortCols));
 part.setParameters(new HashMap<String, String>());
 ms.add_partition(part);
 return part;
}

代码示例来源：origin: apache/hive

@Override
public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats,
  List<String> partVals, String validWriteIds, long writeId)
  throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
 Map<String, String> newParams = rawStore.updatePartitionColumnStatistics(
   colStats, partVals, validWriteIds, writeId);
 // in case of event based cache update, cache is updated during commit txn
 if (newParams != null && !canUseEvents) {
  String catName = colStats.getStatsDesc().isSetCatName() ?
    normalizeIdentifier(colStats.getStatsDesc().getCatName()) : DEFAULT_CATALOG_NAME;
  String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName());
  String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName());
  if (!shouldCacheTable(catName, dbName, tblName)) {
   return newParams;
  }
  Partition part = getPartition(catName, dbName, tblName, partVals);
  part.setParameters(newParams);
  sharedCache.alterPartitionInCache(catName, dbName, tblName, partVals, part);
  sharedCache.updatePartitionColStatsInCache(catName, dbName, tblName, partVals, colStats.getStatsObj());
 }
 return newParams;
}

代码示例来源：origin: apache/hive

private static Partition makePartitionObject(String dbName, String tblName,
  List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException {
 Partition part4 = new Partition();
 part4.setDbName(dbName);
 part4.setTableName(tblName);
 part4.setValues(ptnVals);
 part4.setParameters(new HashMap<>());
 part4.setSd(tbl.getSd().deepCopy());
 part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
 part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix);
 MetaStoreServerUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true);
 return part4;
}

代码示例来源：origin: apache/hive

private void add_partition(HiveMetaStoreClient client, Table table,
  List<String> vals, String location) throws TException {
 Partition part = new Partition();
 part.setDbName(table.getDbName());
 part.setTableName(table.getTableName());
 part.setValues(vals);
 part.setParameters(new HashMap<>());
 part.setSd(table.getSd().deepCopy());
 part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
 part.getSd().setLocation(table.getSd().getLocation() + location);
 client.add_partition(part);
}

代码示例来源：origin: apache/hive

private void addPartition(HiveMetaStoreClient client, Table table,
   List<String> vals, String location) throws TException {

  Partition part = new Partition();
  part.setDbName(table.getDbName());
  part.setTableName(table.getTableName());
  part.setValues(vals);
  part.setParameters(new HashMap<String, String>());
  part.setSd(table.getSd().deepCopy());
  part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
  part.getSd().setLocation(table.getSd().getLocation() + location);

  client.add_partition(part);
 }
}

内容来源于网络，如有侵权，请联系作者删除！

Partition

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多

    Partition类方法

    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com