com.addthis.hydra.data.query.Query类的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(12.3k)|赞(0)|评价(0)|浏览(102)

本文整理了Java中com.addthis.hydra.data.query.Query类的一些代码示例,展示了Query类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Query类的具体详情如下:
包路径:com.addthis.hydra.data.query.Query
类名称:Query

Query介绍

[英]Object representation of a tree query.
[中]树查询的对象表示。

代码示例

代码示例来源:origin: addthis/hydra

@Test
  public void pipeline() {
    String path = "+:+hits,+nodes$+foo=123/+/++123/+%top=hit/a,b,c/|foo/|+bar/*/+%goo/(1-5)+";
    String[] ops = {"sort"};
    Query q = new Query("job", new String[] { path }, ops);
    Query subQ = q.createPipelinedQuery();
    System.out.println(subQ.toString());
  }
}

代码示例来源:origin: addthis/hydra

public QueryEntryInfo getStat() {
  QueryEntryInfo stat = new QueryEntryInfo();
  stat.paths = query.getPaths();
  stat.uuid = query.queryId();
  stat.ops = opsLog;
  stat.job = query.getJob();
  stat.sources = query.getParameter("sources");
  stat.remoteip = query.getParameter("remoteip");
  stat.sender = query.getParameter("sender");
  stat.lines = preOpLines.get();
  stat.sentLines = postOpLines.get();
  stat.runTime = getRunTime();
  stat.startTime = startTime;
  stat.tasks = lastSourceInfo;
  stat.state = queryState;
  return stat;
}

代码示例来源:origin: addthis/hydra

void checkForStragglersMeans() {
  int totalTasks = sourceAggregator.totalTasks;
  int numRemaining = totalTasks - sourceAggregator.completed;
  int tasksDoneCutoff = Math.max(1, (int) Math.ceil(AggregateConfig.stragglerCheckHostPercentage * totalTasks));
  long elapsedTime = JitterClock.globalTime() - sourceAggregator.startTime;
  double timeCutoff = AggregateConfig.stragglerCheckMeanRuntimeFactor * getMeanRuntime();
  if (numRemaining == 0) {
    if (MeshSourceAggregator.log.isDebugEnabled() || sourceAggregator.query.isTraced()) {
      Query.traceLog.info("Straggler checker for {} detected query done. Exiting.",
                sourceAggregator.query.uuid());
    }
  } else if ((numRemaining <= tasksDoneCutoff) && (elapsedTime > timeCutoff)) {
    handleStragglers();
  }
}

代码示例来源:origin: addthis/hydra

/**
 * @return first a query suitable for the next query worker in the stack
 */
public Query createPipelinedQuery() {
  Query newQuery = cloneTo(new Query());
  if (ops != null && ops.length > 0) {
    String[] newops = new String[ops.length - 1];
    System.arraycopy(ops, 1, newops, 0, newops.length);
    newQuery.ops = newops;
    String pop = ops[0];
    ops = new String[]{pop};
  }
  return newQuery;
}

代码示例来源:origin: addthis/hydra

Query query = new Query(job, new String[]{path}, new String[]{kv.getValue("ops"), kv.getValue("rops")});
query.setTraced(kv.getIntValue("trace", 0) == 1);
query.setParameterIfNotYetSet("hosts", kv.getValue("hosts"));
query.setParameterIfNotYetSet("gate", kv.getValue("gate"));
query.setParameterIfNotYetSet("originalrequest", kv.getValue("originalrequest"));
SocketAddress remoteIP = ctx.channel().remoteAddress();
if (remoteIP instanceof InetSocketAddress) { // only log implementations with known methods
  query.setParameterIfNotYetSet("remoteip", ((InetSocketAddress) remoteIP).getAddress().getHostAddress());
query.setParameterIfNotYetSet("allocator", kv.getValue("allocator"));
query.setParameterIfNotYetSet("allowPartial", kv.getValue("allowPartial"));
query.setParameterIfNotYetSet("tasks", kv.getValue("tasks"));
  query.setParameter("allocator", "lazy");
  query.setParameter("maxSimul", maxSimul);
String gdriveAccessToken = kv.getValue("accesstoken");
int timeout = Math.min(kv.getIntValue("timeout", maxQueryTime), maxQueryTime);
query.setParameterIfNotYetSet("timeout", timeout);
query.setParameter("sender", kv.getValue("sender"));
query.setParameter("injectSource", kv.getValue("injectSource"));
      .put("query.path", query.getPaths()[0])
      .put("query.hosts", query.getParameter("hosts"))
      .put("query.ops", query.getOps())
      .put("trace", query.isTraced())
      .put("sources", query.getParameter("sources"))
      .put("time", System.currentTimeMillis())
      .put("job.id", query.getJob())

代码示例来源:origin: addthis/hydra

@Test
public void testCompact() {
  String path = "+:+hits,+nodes$+foo=123/+/++123/+%top=hit/a,b,c/|foo/|+bar/*/+%goo/(1-5)+";
  Query q = new Query("job", new String[] { path }, null);
  Assert.assertEquals(path, q.getPathString(q.getQueryPaths().get(0)));
}

代码示例来源:origin: addthis/hydra

Query query = new Query(job, paths.toArray(new String[paths.size()]), ops.toArray(new String[ops.size()]));
query.setTraced(traced);
for (Entry<String, String> e : qparam.entrySet()) {
  query.setParameter(e.getKey(), e.getValue());

代码示例来源:origin: addthis/hydra

protected void writeQuery(ChannelHandlerContext ctx, Query query, ChannelPromise promise) throws Exception {
  String[] opsLog = query.getOps();
  Query remoteQuery = query.createPipelinedQuery();
  boolean allowPartial = Boolean.valueOf(query.getParameter("allowPartial"));
  Set<Integer> tasks = parseTasks(query.getParameter("tasks"));
  List<QueryTaskSource[]> sourcesPerDir = new ArrayList<>(2);
  for (String combinedUnresolved : JOB_SPLITTER.split(query.getJob())) {
    String jobIdOrAlias = getJobWithoutSubdirectory(combinedUnresolved);
    String subdirectory = getJobSubdirectory(combinedUnresolved);

代码示例来源:origin: addthis/hydra

Query q = new Query(null,
          new String[]{kv.getValue("query", kv.getValue("path", ""))},
          null);
String qo = "{path:" + kv.getValue("query", kv.getValue("path", "")) + "}";
Query q = CodecJSON.decodeString(Query.class, qo);
writer.write(q.getPaths()[0]);
break;

代码示例来源:origin: addthis/hydra

static void sendTimeout(QueryEntry entry, long timeout) {
    String message = "[timeout.watcher] timeout: " + timeout +
             " has been exceeded, canceling query: " + entry.query.uuid();
     if (!entry.tryFailure(new TimeoutException(message))) {
       log.warn("QUERY TIMEOUT FAILURE: query: {}", entry.query.uuid());
     }
  }
}

代码示例来源:origin: addthis/hydra

protected void writeQuery(final ChannelHandlerContext ctx, Query msg, ChannelPromise promise)
    throws Exception {
  this.requestPromise = promise;
  this.queryUser = new DataChannelOutputToNettyBridge(ctx, promise);
  this.query = msg;
  query.queryPromise = queryPromise;
  // create a processor chain based in query ops terminating the query user
  this.opProcessorConsumer = query.newProcessor(queryUser, opPromise);
  queryEntry = new QueryEntry(query, opsLog, this, aggregator);
  // Check if the uuid is repeated, then make a new one
  if (queryTracker.running.putIfAbsent(query.uuid(), queryEntry) != null) {
    throw new QueryException("Query uuid somehow already in use : " + query.uuid());
  }
  log.debug("Executing.... {} {}", query.uuid(), queryEntry);
  ctx.pipeline().remove(this);
  opPromise.addListener(this);
  queryPromise.addListener(this);
  requestPromise.addListener(this);
  ctx.write(opProcessorConsumer, queryPromise);
}

代码示例来源:origin: addthis/hydra

public String getParameter(String key) {
  return getParameter(key, null);
}

代码示例来源:origin: addthis/hydra

@Before
public void setup() throws Exception {
  query = new Query("jobid", new String[]{"path"}, new String[]{"ops"});
  MockitoAnnotations.initMocks(this);
  QueryTaskSource[] taskSources = new QueryTaskSource[]{taskSource0, taskSource1, taskSource2, taskSource3};
  for (QueryTaskSource x: taskSources) {
    stubSelectedSource(x);
  }
  MeshSourceAggregator underlying = new MeshSourceAggregator(taskSources, null, null, query);
  underlying.queryPromise = mock(ChannelProgressivePromise.class);
  underlying.consumer = mock(DataChannelOutput.class);
  underlying.channelWritable = true;
  underlying.executor = executor;
  sourceAggregator = spy(underlying);
  queryTask = new QueryTask(sourceAggregator);
}

代码示例来源:origin: addthis/hydra

public Query setParameterIfNotYetSet(String key, Object value) {
  if (params.get(key) == null) {
    setParameter(key, value);
  }
  return this;
}

代码示例来源:origin: addthis/hydra

/**
 * Part 1 - SETUP
 * Initialize query run -- parse options, create Query object
 */
protected void setup() throws Exception {
  long startTime = System.currentTimeMillis();
  MeshQuerySource.queueTimes.update(creationTime - startTime, TimeUnit.MILLISECONDS);
  query = CodecJSON.decodeString(Query.class, options.get("query"));
  // set as soon as possible (and especially before creating op processor)
  query.queryPromise = bridge.queryPromise;
  // Parse the query and return a reference to the last QueryOpProcessor.
  ChannelProgressivePromise opPromise =
      new DefaultChannelProgressivePromise(null, ImmediateEventExecutor.INSTANCE);
  queryOpProcessor = query.newProcessor(bridge, opPromise);
}

代码示例来源:origin: addthis/hydra

/**
 * Performs a query search, writes the results to a data channel. This function does not break the execution of the
 * query if the client channel gets closed.
 *
 * @param query    A Query object that contains the path or paths of the root query.
 * @param result   A DataChannelOutput to which the result will be written. In practice, this will be the head of
 *                 a QueryOpProcessor that represents the first operator in a query, which in turn sends its output
 *                 to another QueryOpProcessor and the last will send its output to a DataChannelOutput sending bytes
 *                 back to meshy, usually defined at the MQSource side of code.
 * @param queryPromise A wrapper for a boolean flag that gets set to true by MQSource in case the user
 *                 cancels the query at the MQMaster side.
 */
public void search(Query query, DataChannelOutput result,
    ChannelProgressivePromise queryPromise) throws QueryException {
  for (QueryElement[] path : query.getQueryPaths()) {
    if (!(queryPromise.isDone())) {
      search(path, result, queryPromise);
    }
  }
}

代码示例来源:origin: addthis/hydra

@Override
  public void cancel(String message) {
    log.warn(query.uuid() + " cancel called on handle " + consumer + " message: " + message);
    if (engine != null) {
      interrupt();
    }
  }
}

代码示例来源:origin: addthis/hydra

QueryEntry(Query query, String[] opsLog, TrackerHandler trackerHandler, MeshSourceAggregator aggregator) {
  this.query = query;
  this.opsLog = opsLog;
  this.trackerHandler = trackerHandler;
  this.preOpLines = new AtomicInteger();
  this.postOpLines = new AtomicInteger();
  this.aggregator = aggregator;
  final String timeoutInSeconds = query.getParameter("timeout");
  this.startTime = System.currentTimeMillis();
  if (timeoutInSeconds != null) {
    waitTime = Integer.parseInt(timeoutInSeconds);
  } else {
    waitTime = -1;
  }
}

代码示例来源:origin: addthis/hydra

/**
 * Part 2 - ENGINE CACHE
 * Get a QueryEngine for our query -- check the cache for a suitable candidate, otherwise make one.
 * Most of this logic is handled by the QueryEngineCache.get() function.
 */
protected QueryEngine getEngine() throws Exception {
  final long engineGetStartTime = System.currentTimeMillis();
  // Use the canonical path stored in the canonicalDirString to create a QueryEngine. By that way
  // if the alias changes new queries will use the latest available
  // database and the old engines will be automatically closed after their TTL expires.
  QueryEngine engine = MeshQuerySource.queryEngineCache.getAndLease(goldDirString);
  final long engineGetDuration = System.currentTimeMillis() - engineGetStartTime;
  MeshQuerySource.engineGetTimer.update(engineGetDuration, TimeUnit.MILLISECONDS);
  if (engine == null) //Cache returned null -- this doesn't mean cache miss. It means something went fairly wrong
  {
    log.warn("[QueryReference] Unable to retrieve queryEngine for query: {}, key: {} after waiting: {}ms",
        query.uuid(), goldDirString, engineGetDuration);
    throw new DataChannelError("Unable to retrieve queryEngine for query: " + query.uuid() +
                  ", key: " + goldDirString + " after waiting: " + engineGetDuration + "ms");
  } //else we got an engine so we're good -- maybe this logic should be in the cache get
  if ((engineGetDuration > MeshQuerySource.slowQueryThreshold) || log.isDebugEnabled() || query.isTraced()) {
    Query.traceLog.info(
        "[QueryReference] Retrieved queryEngine for query: {}, key:{} after waiting: {}ms.  slow={}",
        query.uuid(), goldDirString, engineGetDuration,
        engineGetDuration > MeshQuerySource.slowQueryThreshold);
  }
  return engine;
}

代码示例来源:origin: addthis/hydra

@Override
public void run() {
  try {
    long currentTime = System.currentTimeMillis();
    for (QueryEntry queryEntry : running.values()) {
      if (queryEntry.waitTime <= 0) {
        continue;
      }
      long queryDuration = currentTime - queryEntry.startTime;
      // wait time is in seconds
      double queryDurationInSeconds = queryDuration / 1000.0;
      if (queryDurationInSeconds < queryEntry.waitTime) {
        log.info("query: {} running for: {} timeout is: {}",
             queryEntry.query.uuid(), queryDurationInSeconds, queryEntry.waitTime);
      } else {
        log.warn("QUERY TIMEOUT query: {} running for: {} timeout is: {}",
             queryEntry.query.uuid(), queryDurationInSeconds, queryEntry.waitTime);
        // sanity check duration
        if (queryDurationInSeconds > (2 * queryEntry.waitTime)) {
          log.warn("query: {} query duration was insane, resetting to waitTime for logging. startTime: {}",
               queryEntry.query.uuid(), queryEntry.startTime);
        }
        sendTimeout(queryEntry, queryEntry.waitTime);
      }
    }
  } catch (Throwable ex) {
    log.error("error while running timeout watcher; swallowing to keep thread alive", ex);
  }
}

相关文章