backtype.storm.tuple.Tuple.getMessageId()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(11.0k)|赞(0)|评价(0)|浏览(92)

本文整理了Java中backtype.storm.tuple.Tuple.getMessageId()方法的一些代码示例，展示了Tuple.getMessageId()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Tuple.getMessageId()方法的具体详情如下：
包路径：backtype.storm.tuple.Tuple
类名称：Tuple
方法名：getMessageId

Tuple.getMessageId介绍

[英]Gets the message id that associated with this tuple.
[中]获取与此元组关联的消息id。

代码示例

代码示例来源：origin: alibaba/jstorm

/**
 * one of the tuples has been chosen, the logs should be output
 */
private static boolean sample(Collection<Tuple> anchors) {
  if (Double.compare(sampleRate, 1.0d) >= 0)
    return true;
  for (Tuple t : anchors) {
    if (sample(t.getMessageId().getAnchors())) {
      return true;
    }
  }
  return false;
}

代码示例来源：origin: alibaba/jstorm

@Override
public void fail(Tuple input) {
  // if ackerNum == 0, we can just return
  if (input.getMessageId() != null) {
    pendingAcks.remove(input);
    for (Map.Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
      List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey());
      sendBoltMsg(Acker.ACKER_FAIL_STREAM_ID, null, ackTuple, null, null);
    }
  }
  taskStats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId());
}

代码示例来源：origin: alibaba/jstorm

protected MessageId getMessageId(Collection<Tuple> anchors) {
  MessageId ret = null;
  if (anchors != null && ackerNum > 0) {
    Map<Long, Long> anchors_to_ids = new HashMap<>();
    for (Tuple a : anchors) {
      if (a.getMessageId() != null) {
        Long edge_id = MessageId.generateId(random);
        put_xor(pendingAcks, a, edge_id);
        MessageId messageId = a.getMessageId();
        if (messageId != null) {
          for (Long root_id : messageId.getAnchorsToIds().keySet()) {
            put_xor(anchors_to_ids, root_id, edge_id);
          }
        }
      }
    }
    ret = MessageId.makeId(anchors_to_ids);
  }
  return ret;
}

代码示例来源：origin: alibaba/jstorm

@Override
public void fail(Tuple input) {
  // if ackerNum == 0, we can just return
  if (input.getMessageId() != null) {
    pendingAcks.remove(input);
    for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
      unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_FAIL_STREAM_ID,
          JStormUtils.mk_list((Object) e.getKey()));
    }
  }
  tupleStartTimes.remove(input);
  taskStats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId());
}

代码示例来源：origin: alibaba/jstorm

private boolean sendAckTuple(Tuple input) {
  boolean ret = false;
  Integer pendingCount;
  synchronized (pendingTuples) {
    pendingCount = pendingTuples.get(input);
  }
  if (pendingCount == null || pendingCount <= 0) {
    long ack_val = 0L;
    Object pend_val = pendingAcks.remove(input);
    if (pend_val != null) {
      ack_val = (Long) (pend_val);
    }
    MessageId messageId = input.getMessageId();
    if (messageId != null) {
      for (Map.Entry<Long, Long> e : messageId.getAnchorsToIds().entrySet()) {
        List<Object> ackTuple =
            JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val));
        sendBoltMsg(Acker.ACKER_ACK_STREAM_ID, null, ackTuple, null, null);
      }
    }
    ret = true;
  }
  return ret;
}

代码示例来源：origin: alibaba/jstorm

@Override
public void ack(Tuple input) {
  if (input.getMessageId() != null) {
    Long ack_val = 0L;
    Object pend_val = pendingAcks.remove(input);
    if (pend_val != null) {
      ack_val = (Long) (pend_val);
    }
    for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
      unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_ACK_STREAM_ID,
          JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val)));
    }
  }
  Long latencyStart = (Long) tupleStartTimes.remove(input);
  taskStats.bolt_acked_tuple(input.getSourceComponent(), input.getSourceStreamId());
  if (latencyStart != null && JStormMetrics.enabled) {
    long endTime = System.currentTimeMillis();
    taskStats.update_bolt_acked_latency(input.getSourceComponent(), input.getSourceStreamId(),
        latencyStart, endTime);
  }
}

代码示例来源：origin: alibaba/mdrill

public byte[] serialize(Tuple tuple) {
  try {
    _outputter.reset();
    WritableUtils.writeVInt(_dataOutputter, tuple.getSourceTask());
    WritableUtils.writeVInt(_dataOutputter, _ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()));
    tuple.getMessageId().serialize(_dataOutputter);
    _kryo.serializeInto(tuple.getValues(), _outputter);
    return _outputter.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

代码示例来源：origin: alibaba/jstorm

private void processTupleEvent(Tuple tuple) {
  if (tuple.getMessageId() != null && tuple.getMessageId().isAnchored()) {
    tupleStartTimes.put(tuple, System.currentTimeMillis());
  }
  try {
    // for watermarks, just forward to downstream operators
    if (!isSystemBolt && Common.WATERMARK_STREAM_ID.equals(tuple.getSourceStreamId())) {
      outputCollector.emit(Common.WATERMARK_STREAM_ID, tuple.getValues());
    }
    if (!isSystemBolt && tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) {
      TopoMasterCtrlEvent event = (TopoMasterCtrlEvent) tuple.getValue(0);
      if (event.isTransactionEvent()) {
        bolt.execute(tuple);
      } else {
        LOG.warn("Received an unexpected control event, {}", event);
      }
    } else if (tuple.getSourceStreamId().equals(Common.TOPOLOGY_MASTER_REGISTER_METRICS_RESP_STREAM_ID)) {
      this.metricsReporter.updateMetricMeta((Map<String, Long>) tuple.getValue(0));
    } else {
      bolt.execute(tuple);
    }
  } catch (Throwable e) {
    error = e;
    LOG.error("bolt execute error ", e);
    reportError.report(e);
  }
}

代码示例来源：origin: alibaba/jstorm

@Override
public void ack(Tuple input) {
  if (input.getMessageId() != null) {
    if (!sendAckTuple(input)) {
      pendingAckQueue.add(input);
    }
  }
  Long latencyStart = (Long) tupleStartTimes.remove(input);
  taskStats.bolt_acked_tuple(input.getSourceComponent(), input.getSourceStreamId());
  if (latencyStart != null && JStormMetrics.enabled) {
    long endTime = System.currentTimeMillis();
    taskStats.update_bolt_acked_latency(input.getSourceComponent(), input.getSourceStreamId(), latencyStart, endTime);
  }
}

代码示例来源：origin: alibaba/jstorm

@Override
protected MessageId getMessageId(Collection<Tuple> anchors) {
  MessageId ret = null;
  if (anchors != null) {
    Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>();
    long now = System.currentTimeMillis();
    if (now - lastRotate > rotateTime) {
      pendingAcks.rotate();
      synchronized (pendingTuples) {
        pendingTuples.rotate();
      }
      lastRotate = now;
    }
    for (Tuple a : anchors) {
      // Long edge_id = MessageId.generateId();
      Long edge_id = MessageId.generateId(random);
      synchronized (pendingAcks) {
        put_xor(pendingAcks, a, edge_id);
      }
      MessageId messageId = a.getMessageId();
      if (messageId != null) {
        for (Long root_id : messageId.getAnchorsToIds().keySet()) {
          put_xor(anchors_to_ids, root_id, edge_id);
        }
      }
    }
    ret = MessageId.makeId(anchors_to_ids);
  }
  return ret;
}

代码示例来源：origin: alibaba/mdrill

@Override
public void fail(Tuple input) {
pending_acks.remove(input);
for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
  UnanchoredSend.send(topologyContext, sendTargets,
    workerTransfer, Acker.ACKER_FAIL_STREAM_ID,
    StormUtils.mk_list((Object) e.getKey()));
}
  Long delta = TasksCommon.tuple_time_delta(tuple_start_times,input);
  if (delta != null) {
  Stats.bolt_failed_tuple((BoltTaskStatsRolling) task_stats,
    input.getSourceComponent(), input.getSourceStreamId(),
    delta);
  }
}

代码示例来源：origin: alibaba/jstorm

public void deserializeTuple(KryoTupleDeserializer deserializer, byte[] serMsg, DisruptorQueue queue) {
    Tuple tuple = deserializer.deserialize(serMsg);
    if (tuple != null) {
      if (JStormDebugger.isDebugRecv(tuple.getMessageId())) {
        LOG.info(idStr + " receive " + tuple.toString());
      }
      //queue.publish(tuple);
      if (isBackpressureEnable) {
        if (backpressureStatus) {
          while (queue.pctFull() > lowMark) {
            JStormUtils.sleepMs(1);
          }
          queue.publish(tuple);
          backpressureStatus = false;
        } else {
          queue.publish(tuple);
          if (queue.pctFull() > highMark) {
            backpressureStatus = true;
          }
        }
      } else {
        queue.publish(tuple);
      }
    }
  }
}

代码示例来源：origin: alibaba/mdrill

@Override
public void ack(Tuple input) {
Object ack_val = pending_acks.remove(input);
if (ack_val == null) {
  ack_val = 0l;
}
for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
  UnanchoredSend.send(
    topologyContext,
    sendTargets,
    workerTransfer,
    Acker.ACKER_ACK_STREAM_ID,
    StormUtils.mk_list((Object)e.getKey(),StormUtils.bit_xor(e.getValue(), ack_val)));
  }
    Long delta = TasksCommon.tuple_time_delta(tuple_start_times, input);
    if (delta != null) {
      Stats.bolt_acked_tuple((BoltTaskStatsRolling) task_stats,
        input.getSourceComponent(), input.getSourceStreamId(),
        delta);
    }
}

代码示例来源：origin: alibaba/mdrill

private List<Integer> boltEmit(String out_stream_id,Collection<Tuple> anchors, List<Object> values, Integer out_task_id) {
try {
  java.util.Set<Integer> out_tasks = null;
  if (out_task_id != null) {
  out_tasks = sendTargets.get(out_task_id, out_stream_id, values);
  } else {
  out_tasks = sendTargets.get(out_stream_id, values);
  }
  for (Integer t : out_tasks) {
  HashMap anchors_to_ids = new HashMap();
  if (anchors != null) {
    for (Tuple a : anchors) {
    Long edge_id = MessageId.generateId();
    TasksCommon.put_xor(pending_acks, a, edge_id);
    for (Long root_id : a.getMessageId().getAnchorsToIds().keySet()) {
      TasksCommon.put_xor(anchors_to_ids, root_id, edge_id);
    }
    }
  }
  MessageId msgid=MessageId.makeId(anchors_to_ids);
  workerTransfer.transfer(t,new Tuple(topologyContext, values, task_id,out_stream_id, msgid));
  }
  return StormUtils.mk_list(out_tasks);
} catch (Exception e) {
  LOG.error("bolt emit", e);
}
return new ArrayList<Integer>();
}

代码示例来源：origin: alibaba/jstorm

MessageId msgId = tuple.getMessageId();
if (msgId != null) {
  msgId.serialize(output);

代码示例来源：origin: com.alibaba.jstorm/jstorm-core

@Override
public void fail(Tuple input) {
  // if ackerNum == 0, we can just return
  if (input.getMessageId() != null) {
    pending_acks.remove(input);
    for (Map.Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
      List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey());
      sendBoltMsg(Acker.ACKER_FAIL_STREAM_ID, null, ackTuple, null, null);
    }
  }
  task_stats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId());
}

代码示例来源：origin: com.alibaba.jstorm/jstorm-core

@Override
public void fail(Tuple input) {
  // if ackerNum == 0, we can just return
  if (input.getMessageId() != null) {
    pending_acks.remove(input);
    for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds().entrySet()) {
      unanchoredSend(topologyContext, sendTargets, taskTransfer, Acker.ACKER_FAIL_STREAM_ID, JStormUtils.mk_list((Object) e.getKey()));
    }
  }
  tuple_start_times.remove(input);
  task_stats.bolt_failed_tuple(input.getSourceComponent(), input.getSourceStreamId());
}

代码示例来源：origin: dongeforever/middlewarerace

@Override
public void execute(Tuple tuple) {
  String sentence = tuple.getString(0);
  for (String word : sentence.split("\\s+")) {
    //锚定在这个位置
    collector.emit(tuple,new Values(word));
  }
  LOG.info("SPLIT_SENTENCE:{} msgId:{}",sentence, tuple.getMessageId().getAnchors());
  collector.ack(tuple);
}

代码示例来源：origin: mayconbordin/storm-applications

@Override
public String format(Tuple tuple) {
  Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId());
  String values = "";
  for (int i=0; i<tuple.size(); i++) {
    if (i != 0) values += ", ";
    values += String.format("%s=%s", schema.get(i), tuple.getValue(i));
  }
  
  return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(),
      tuple.getSourceStreamId(), tuple.getMessageId().toString(), values);
}

代码示例来源：origin: com.n3twork.storm/storm-core

public byte[] serialize(Tuple tuple) {
  try {
    
    _kryoOut.clear();
    _kryoOut.writeInt(tuple.getSourceTask(), true);
    _kryoOut.writeInt(_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true);
    tuple.getMessageId().serialize(_kryoOut);
    _kryo.serializeInto(tuple.getValues(), _kryoOut);
    return _kryoOut.toBytes();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}

内容来源于网络，如有侵权，请联系作者删除！

Tuple

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com