kafka.common.TopicAndPartition.<init>()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(10.9k)|赞(0)|评价(0)|浏览(103)

本文整理了Java中kafka.common.TopicAndPartition.<init>()方法的一些代码示例,展示了TopicAndPartition.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。TopicAndPartition.<init>()方法的具体详情如下:
包路径:kafka.common.TopicAndPartition
类名称:TopicAndPartition
方法名:<init>

TopicAndPartition.<init>介绍

暂无

代码示例

代码示例来源:origin: apache/flink

@Override
protected TopicAndPartition createKafkaPartitionHandle(KafkaTopicPartition partition) {
  return new TopicAndPartition(partition.getTopic(), partition.getPartition());
}

代码示例来源:origin: apache/incubator-pinot

@Override
public short errorCode(String topic, int partition) {
 TopicAndPartition key = new TopicAndPartition(topic, partition);
 if (errorMap.containsKey(key)) {
  return errorMap.get(key);
 } else {
  return Errors.NONE.code();
 }
}

代码示例来源:origin: apache/incubator-pinot

@Override
public ByteBufferMessageSet messageSet(String topic, int partition) {
 if (errorMap.containsKey(new TopicAndPartition(topic, partition))) {
  throw new IllegalArgumentException();
 } else {
  // TODO Maybe generate dummy messages here?
  return new ByteBufferMessageSet(Collections.<Message>emptyList());
 }
}

代码示例来源:origin: apache/incubator-gobblin

private FetchRequest createFetchRequest(KafkaPartition partition, long nextOffset) {
 TopicAndPartition topicAndPartition = new TopicAndPartition(partition.getTopicName(), partition.getId());
 PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(nextOffset, this.bufferSize);
 Map<TopicAndPartition, PartitionFetchInfo> fetchInfo =
   Collections.singletonMap(topicAndPartition, partitionFetchInfo);
 return new FetchRequest(this.fetchCorrelationId, this.clientName, this.fetchTimeoutMillis, this.fetchMinBytes,
   fetchInfo);
}

代码示例来源:origin: apache/incubator-gobblin

private FetchRequest createFetchRequest(KafkaPartition partition, long nextOffset) {
 TopicAndPartition topicAndPartition = new TopicAndPartition(partition.getTopicName(), partition.getId());
 PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(nextOffset, this.bufferSize);
 Map<TopicAndPartition, PartitionFetchInfo> fetchInfo =
   Collections.singletonMap(topicAndPartition, partitionFetchInfo);
 return new FetchRequest(this.fetchCorrelationId, this.clientName,
   this.fetchTimeoutMillis, this.fetchMinBytes, fetchInfo);
}

代码示例来源:origin: apache/incubator-gobblin

@Override
protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
 Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
   Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
     new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
 return getOffset(partition, offsetRequestInfo);
}

代码示例来源:origin: alibaba/jstorm

public long getOffset(String topic, int partition, long startOffsetTime) {
  SimpleConsumer simpleConsumer = findLeaderConsumer(partition);
  if (simpleConsumer == null) {
    LOG.error("Error consumer is null get offset from partition:" + partition);
    return -1;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
  long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
  if (offsets.length > 0) {
    return offsets[0];
  } else {
    return NO_OFFSET;
  }
}

代码示例来源:origin: apache/incubator-gobblin

@Override
protected long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
 Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
   Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
     new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
 return getOffset(partition, offsetRequestInfo);
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
 Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
   Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
     new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
 return getOffset(partition, offsetRequestInfo);
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
 Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
   Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
     new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
 return getOffset(partition, offsetRequestInfo);
}

代码示例来源:origin: prestodb/presto

private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId)
{
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
  // The API implies that this will always return all of the offsets. So it seems a partition can not have
  // more than Integer.MAX_VALUE-1 segments.
  //
  // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
  // should not be 0.
  PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE);
  OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
  if (offsetResponse.hasError()) {
    short errorCode = offsetResponse.errorCode(topicName, partitionId);
    throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'");
  }
  return offsetResponse.offsets(topicName, partitionId);
}

代码示例来源:origin: linkedin/cruise-control

newReplicaAssignment.put(new TopicAndPartition(topic, pm.partition()),
             JavaConverters.asScalaIteratorConverter(newAssignedReplica.iterator()).asScala().toSeq());

代码示例来源:origin: Graylog2/graylog2-server

JODA_TIME);
final TopicAndPartition topicAndPartition = new TopicAndPartition("messagejournal", 0);
final Option<Log> messageLog = logManager.getLog(topicAndPartition);
if (messageLog.isEmpty()) {

代码示例来源:origin: apache/incubator-pinot

OffsetRequest request = new OffsetRequest(Collections.singletonMap(new TopicAndPartition(_topic, _partition),
  new PartitionOffsetRequestInfo(offsetRequestTime, 1)), kafka.api.OffsetRequest.CurrentVersion(), _clientId);
OffsetResponse offsetResponse;

代码示例来源:origin: apache/incubator-druid

private long getOffset(boolean earliest) throws InterruptedException
{
 TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
 Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
 requestInfo.put(
   topicAndPartition,
   new PartitionOffsetRequestInfo(
     earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1
   )
 );
 OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
 OffsetResponse response;
 try {
  response = consumer.getOffsetsBefore(request);
 }
 catch (Exception e) {
  ensureNotInterrupted(e);
  log.error(e, "caught exception in getOffsetsBefore [%s] - [%s]", topic, partitionId);
  return -1;
 }
 if (response.hasError()) {
  log.error(
    "error fetching data Offset from the Broker [%s]. reason: [%s]", leaderBroker.host(),
    response.errorCode(topic, partitionId)
  );
  return -1;
 }
 long[] offsets = response.offsets(topic, partitionId);
 return earliest ? offsets[0] : offsets[offsets.length - 1];
}

代码示例来源:origin: pinterest/secor

private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
      topicPartition.getPartition());
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
      kafka.api.OffsetRequest.LatestTime(), 1));
  final String clientName = getClientName(topicPartition);
  OffsetRequest request = new OffsetRequest(requestInfo,
                       kafka.api.OffsetRequest.CurrentVersion(),
                       clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);
  if (response.hasError()) {
    throw new RuntimeException("Error fetching offset data. Reason: " +
        response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
  }
  long[] offsets = response.offsets(topicPartition.getTopic(),
      topicPartition.getPartition());
  return offsets[0] - 1;
}

代码示例来源:origin: uber/chaperone

for (Map.Entry<Integer, Long> offsetEntry : topicEntry.getValue().entrySet()) {
 long commitOffset = offsetEntry.getValue();
 TopicAndPartition tp = new TopicAndPartition(topicName, offsetEntry.getKey());
 Long latestOffset = partitionLatestOffset.get(tp);
 if (latestOffset != null) {

代码示例来源:origin: uber/chaperone

private void updateLeaderMap() {
 for (String broker : brokerList) {
  try {
   SimpleConsumer consumer = getSimpleConsumer(broker);
   TopicMetadataRequest req = new TopicMetadataRequest(auditTopics);
   kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
   List<TopicMetadata> metaData = resp.topicsMetadata();
   for (TopicMetadata tmd : metaData) {
    for (PartitionMetadata pmd : tmd.partitionsMetadata()) {
     TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId());
     partitionLeader.put(topicAndPartition, getHostPort(pmd.leader()));
    }
   }
  } catch (Exception e) {
   logger.warn("Got exception to get metadata from broker=" + broker, e);
  }
 }
}

代码示例来源:origin: linkedin/camus

TopicAndPartition topicAndPartition = new TopicAndPartition(kafkaRequest.getTopic(), kafkaRequest.getPartition());
log.debug("\nAsking for offset : " + (currentOffset));
PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(currentOffset, fetchBufferSize);

代码示例来源:origin: rakam-io/rakam

private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
  // The API implies that this will always return all of the offsets. So it seems a partition can not have
  // more than Integer.MAX_VALUE-1 segments.
  //
  // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
  // should not be 0.
  PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 10000);
  OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
  if (offsetResponse.hasError()) {
    short errorCode = offsetResponse.errorCode(topicName, partitionId);
    LOGGER.warn(format("Offset response has error: %d", errorCode));
    throw new RakamException("could not fetch data from Kafka, error code is '" + errorCode + "'", HttpResponseStatus.INTERNAL_SERVER_ERROR);
  }
  long[] offsets = offsetResponse.offsets(topicName, partitionId);
  return offsets;
}

相关文章

微信公众号

最新文章

更多