本文整理了Java中kafka.common.TopicAndPartition.<init>()
方法的一些代码示例,展示了TopicAndPartition.<init>()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。TopicAndPartition.<init>()
方法的具体详情如下:
包路径:kafka.common.TopicAndPartition
类名称:TopicAndPartition
方法名:<init>
暂无
代码示例来源:origin: apache/flink
@Override
protected TopicAndPartition createKafkaPartitionHandle(KafkaTopicPartition partition) {
return new TopicAndPartition(partition.getTopic(), partition.getPartition());
}
代码示例来源:origin: apache/incubator-pinot
@Override
public short errorCode(String topic, int partition) {
TopicAndPartition key = new TopicAndPartition(topic, partition);
if (errorMap.containsKey(key)) {
return errorMap.get(key);
} else {
return Errors.NONE.code();
}
}
代码示例来源:origin: apache/incubator-pinot
@Override
public ByteBufferMessageSet messageSet(String topic, int partition) {
if (errorMap.containsKey(new TopicAndPartition(topic, partition))) {
throw new IllegalArgumentException();
} else {
// TODO Maybe generate dummy messages here?
return new ByteBufferMessageSet(Collections.<Message>emptyList());
}
}
代码示例来源:origin: apache/incubator-gobblin
private FetchRequest createFetchRequest(KafkaPartition partition, long nextOffset) {
TopicAndPartition topicAndPartition = new TopicAndPartition(partition.getTopicName(), partition.getId());
PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(nextOffset, this.bufferSize);
Map<TopicAndPartition, PartitionFetchInfo> fetchInfo =
Collections.singletonMap(topicAndPartition, partitionFetchInfo);
return new FetchRequest(this.fetchCorrelationId, this.clientName, this.fetchTimeoutMillis, this.fetchMinBytes,
fetchInfo);
}
代码示例来源:origin: apache/incubator-gobblin
private FetchRequest createFetchRequest(KafkaPartition partition, long nextOffset) {
TopicAndPartition topicAndPartition = new TopicAndPartition(partition.getTopicName(), partition.getId());
PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(nextOffset, this.bufferSize);
Map<TopicAndPartition, PartitionFetchInfo> fetchInfo =
Collections.singletonMap(topicAndPartition, partitionFetchInfo);
return new FetchRequest(this.fetchCorrelationId, this.clientName,
this.fetchTimeoutMillis, this.fetchMinBytes, fetchInfo);
}
代码示例来源:origin: apache/incubator-gobblin
@Override
protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
return getOffset(partition, offsetRequestInfo);
}
代码示例来源:origin: alibaba/jstorm
public long getOffset(String topic, int partition, long startOffsetTime) {
SimpleConsumer simpleConsumer = findLeaderConsumer(partition);
if (simpleConsumer == null) {
LOG.error("Error consumer is null get offset from partition:" + partition);
return -1;
}
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());
long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
if (offsets.length > 0) {
return offsets[0];
} else {
return NO_OFFSET;
}
}
代码示例来源:origin: apache/incubator-gobblin
@Override
protected long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
return getOffset(partition, offsetRequestInfo);
}
代码示例来源:origin: apache/incubator-gobblin
@Override
public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
return getOffset(partition, offsetRequestInfo);
}
代码示例来源:origin: apache/incubator-gobblin
@Override
public long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
return getOffset(partition, offsetRequestInfo);
}
代码示例来源:origin: prestodb/presto
private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId)
{
TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
// The API implies that this will always return all of the offsets. So it seems a partition can not have
// more than Integer.MAX_VALUE-1 segments.
//
// This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
// should not be 0.
PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE);
OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
if (offsetResponse.hasError()) {
short errorCode = offsetResponse.errorCode(topicName, partitionId);
throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'");
}
return offsetResponse.offsets(topicName, partitionId);
}
代码示例来源:origin: linkedin/cruise-control
newReplicaAssignment.put(new TopicAndPartition(topic, pm.partition()),
JavaConverters.asScalaIteratorConverter(newAssignedReplica.iterator()).asScala().toSeq());
代码示例来源:origin: Graylog2/graylog2-server
JODA_TIME);
final TopicAndPartition topicAndPartition = new TopicAndPartition("messagejournal", 0);
final Option<Log> messageLog = logManager.getLog(topicAndPartition);
if (messageLog.isEmpty()) {
代码示例来源:origin: apache/incubator-pinot
OffsetRequest request = new OffsetRequest(Collections.singletonMap(new TopicAndPartition(_topic, _partition),
new PartitionOffsetRequestInfo(offsetRequestTime, 1)), kafka.api.OffsetRequest.CurrentVersion(), _clientId);
OffsetResponse offsetResponse;
代码示例来源:origin: apache/incubator-druid
private long getOffset(boolean earliest) throws InterruptedException
{
TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(
topicAndPartition,
new PartitionOffsetRequestInfo(
earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1
)
);
OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
OffsetResponse response;
try {
response = consumer.getOffsetsBefore(request);
}
catch (Exception e) {
ensureNotInterrupted(e);
log.error(e, "caught exception in getOffsetsBefore [%s] - [%s]", topic, partitionId);
return -1;
}
if (response.hasError()) {
log.error(
"error fetching data Offset from the Broker [%s]. reason: [%s]", leaderBroker.host(),
response.errorCode(topic, partitionId)
);
return -1;
}
long[] offsets = response.offsets(topic, partitionId);
return earliest ? offsets[0] : offsets[offsets.length - 1];
}
代码示例来源:origin: pinterest/secor
private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
topicPartition.getPartition());
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
kafka.api.OffsetRequest.LatestTime(), 1));
final String clientName = getClientName(topicPartition);
OffsetRequest request = new OffsetRequest(requestInfo,
kafka.api.OffsetRequest.CurrentVersion(),
clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if (response.hasError()) {
throw new RuntimeException("Error fetching offset data. Reason: " +
response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
}
long[] offsets = response.offsets(topicPartition.getTopic(),
topicPartition.getPartition());
return offsets[0] - 1;
}
代码示例来源:origin: uber/chaperone
for (Map.Entry<Integer, Long> offsetEntry : topicEntry.getValue().entrySet()) {
long commitOffset = offsetEntry.getValue();
TopicAndPartition tp = new TopicAndPartition(topicName, offsetEntry.getKey());
Long latestOffset = partitionLatestOffset.get(tp);
if (latestOffset != null) {
代码示例来源:origin: uber/chaperone
private void updateLeaderMap() {
for (String broker : brokerList) {
try {
SimpleConsumer consumer = getSimpleConsumer(broker);
TopicMetadataRequest req = new TopicMetadataRequest(auditTopics);
kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
List<TopicMetadata> metaData = resp.topicsMetadata();
for (TopicMetadata tmd : metaData) {
for (PartitionMetadata pmd : tmd.partitionsMetadata()) {
TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId());
partitionLeader.put(topicAndPartition, getHostPort(pmd.leader()));
}
}
} catch (Exception e) {
logger.warn("Got exception to get metadata from broker=" + broker, e);
}
}
}
代码示例来源:origin: linkedin/camus
TopicAndPartition topicAndPartition = new TopicAndPartition(kafkaRequest.getTopic(), kafkaRequest.getPartition());
log.debug("\nAsking for offset : " + (currentOffset));
PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(currentOffset, fetchBufferSize);
代码示例来源:origin: rakam-io/rakam
private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) {
TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);
// The API implies that this will always return all of the offsets. So it seems a partition can not have
// more than Integer.MAX_VALUE-1 segments.
//
// This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
// should not be 0.
PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 10000);
OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);
if (offsetResponse.hasError()) {
short errorCode = offsetResponse.errorCode(topicName, partitionId);
LOGGER.warn(format("Offset response has error: %d", errorCode));
throw new RakamException("could not fetch data from Kafka, error code is '" + errorCode + "'", HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
long[] offsets = offsetResponse.offsets(topicName, partitionId);
return offsets;
}
内容来源于网络,如有侵权,请联系作者删除!