org.apache.flink.api.java.Utils类的使用及代码示例

x33g5p2x  于2022-02-01 转载在 其他  
字(12.1k)|赞(0)|评价(0)|浏览(97)

本文整理了Java中org.apache.flink.api.java.Utils类的一些代码示例,展示了Utils类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Utils类的具体详情如下:
包路径:org.apache.flink.api.java.Utils
类名称:Utils

Utils介绍

[英]Utility class that contains helper methods to work with Java APIs.
[中]

代码示例

代码示例来源:origin: apache/flink

public static String getCallLocationName() {
  return getCallLocationName(4);
}

代码示例来源:origin: apache/flink

private static <T> String getSerializerTree(TypeInformation<T> ti, int indent) {
  String ret = "";
  if (ti instanceof CompositeType) {
    ret += StringUtils.repeat(' ', indent) + ti.getClass().getSimpleName() + "\n";
    CompositeType<T> cti = (CompositeType<T>) ti;
    String[] fieldNames = cti.getFieldNames();
    for (int i = 0; i < cti.getArity(); i++) {
      TypeInformation<?> fieldType = cti.getTypeAt(i);
      ret += StringUtils.repeat(' ', indent + 2) + fieldNames[i] + ":" + getSerializerTree(fieldType, indent);
    }
  } else {
    if (ti instanceof GenericTypeInfo) {
      ret += StringUtils.repeat(' ', indent) + "GenericTypeInfo (" + ti.getTypeClass().getSimpleName() + ")\n";
      ret += getGenericTypeTree(ti.getTypeClass(), indent + 4);
    } else {
      ret += StringUtils.repeat(' ', indent) + ti.toString() + "\n";
    }
  }
  return ret;
}

代码示例来源:origin: apache/flink

/**
 * Debugging utility to understand the hierarchy of serializers created by the Java API.
 * Tested in GroupReduceITCase.testGroupByGenericType()
 */
public static <T> String getSerializerTree(TypeInformation<T> ti) {
  return getSerializerTree(ti, 0);
}

代码示例来源:origin: apache/flink

private static String getGenericTypeTree(Class<?> type, int indent) {
  String ret = "";
  for (Field field : type.getDeclaredFields()) {
    if (Modifier.isStatic(field.getModifiers()) || Modifier.isTransient(field.getModifiers())) {
      continue;
    }
    ret += StringUtils.repeat(' ', indent) + field.getName() + ":" + field.getType().getName() +
      (field.getType().isEnum() ? " (is enum)" : "") + "\n";
    if (!field.getType().isPrimitive()) {
      ret += getGenericTypeTree(field.getType(), indent + 4);
    }
  }
  return ret;
}

代码示例来源:origin: apache/flink

/**
 * Syntactic sugar for aggregate (MAX, field).
 * @param field The index of the Tuple field on which the aggregation function is applied.
 * @return An AggregateOperator that represents the max'ed DataSet.
 *
 * @see org.apache.flink.api.java.operators.AggregateOperator
 */
public AggregateOperator<T> max (int field) {
  return this.aggregate (Aggregations.MAX, field, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Locally sorts the partitions of the DataSet on the specified field in the specified order.
 * DataSet can be sorted on multiple fields by chaining sortPartition() calls.
 *
 * @param field The field index on which the DataSet is sorted.
 * @param order The order in which the DataSet is sorted.
 * @return The DataSet with sorted local partitions.
 */
public SortPartitionOperator<T> sortPartition(int field, Order order) {
  return new SortPartitionOperator<>(this, field, order, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Syntactic sugar for aggregate (MIN, field).
 * @param field The index of the Tuple field on which the aggregation function is applied.
 * @return An AggregateOperator that represents the min'ed DataSet.
 *
 * @see org.apache.flink.api.java.operators.AggregateOperator
 */
public AggregateOperator<T> min (int field) {
  return this.aggregate (Aggregations.MIN, field, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Enforces a re-balancing of the DataSet, i.e., the DataSet is evenly distributed over all parallel instances of the
 * following task. This can help to improve performance in case of heavy data skew and compute intensive operations.
 *
 * <p><b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 *
 * @return The re-balanced DataSet.
 */
public PartitionOperator<T> rebalance() {
  return new PartitionOperator<>(this, PartitionMethod.REBALANCE, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Locally sorts the partitions of the DataSet on the specified field in the specified order.
 * DataSet can be sorted on multiple fields by chaining sortPartition() calls.
 *
 * @param field The field expression referring to the field on which the DataSet is sorted.
 * @param order The order in which the DataSet is sorted.
 * @return The DataSet with sorted local partitions.
 */
public SortPartitionOperator<T> sortPartition(String field, Order order) {
  return new SortPartitionOperator<>(this, field, order, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Creates a union of this DataSet with an other DataSet. The other DataSet must be of the same data type.
 *
 * @param other The other DataSet which is unioned with the current DataSet.
 * @return The resulting DataSet.
 */
public UnionOperator<T> union(DataSet<T> other){
  return new UnionOperator<>(this, other, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Returns a distinct set of a {@link DataSet}.
 *
 * <p>If the input is a {@link org.apache.flink.api.common.typeutils.CompositeType} (Tuple or Pojo type),
 * distinct is performed on all fields and each field must be a key type
 *
 * @return A DistinctOperator that represents the distinct DataSet.
 */
public DistinctOperator<T> distinct() {
  return new DistinctOperator<>(this, null, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Syntactic sugar for aggregate (SUM, field).
 * @param field The index of the Tuple field on which the aggregation function is applied.
 * @return An AggregateOperator that represents the summed DataSet.
 *
 * @see org.apache.flink.api.java.operators.AggregateOperator
 */
public AggregateOperator<T> sum (int field) {
  return this.aggregate (Aggregations.SUM, field, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Creates a DataSet from the given non-empty collection. Note that this operation will result
 * in a non-parallel data source, i.e. a data source with a parallelism of one.
 *
 * <p>The returned DataSet is typed to the given TypeInformation.
 *
 * @param data The collection of elements to create the data set from.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the given collection.
 *
 * @see #fromCollection(Collection)
 */
public <X> DataSource<X> fromCollection(Collection<X> data, TypeInformation<X> type) {
  return fromCollection(data, type, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 *
 * <p>Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type information.
 * This method is useful for cases where the type is generic. In that case, the type class
 * (as given in {@link #fromParallelCollection(SplittableIterator, Class)} does not supply all type information.
 *
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the elements in the iterator.
 *
 * @see #fromParallelCollection(SplittableIterator, Class)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type) {
  return fromParallelCollection(iterator, type, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Applies an Aggregate transformation on a grouped {@link org.apache.flink.api.java.tuple.Tuple} {@link DataSet}.
 *
 * <p><b>Note: Only Tuple DataSets can be aggregated.</b>
 * The transformation applies a built-in {@link Aggregations Aggregation} on a specified field
 *   of a Tuple group. Additional aggregation functions can be added to the resulting
 *   {@link AggregateOperator} by calling {@link AggregateOperator#and(Aggregations, int)}.
 *
 * @param agg The built-in aggregation function that is computed.
 * @param field The index of the Tuple field on which the aggregation function is applied.
 * @return An AggregateOperator that represents the aggregated DataSet.
 *
 * @see org.apache.flink.api.java.tuple.Tuple
 * @see Aggregations
 * @see AggregateOperator
 * @see DataSet
 */
public AggregateOperator<T> aggregate(Aggregations agg, int field) {
  return aggregate(agg, field, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Applies an Aggregate transformation on a non-grouped {@link Tuple} {@link DataSet}.
 *
 * <p><b>Note: Only Tuple DataSets can be aggregated.</b>
 * The transformation applies a built-in {@link Aggregations Aggregation} on a specified field
 *   of a Tuple DataSet. Additional aggregation functions can be added to the resulting
 *   {@link AggregateOperator} by calling {@link AggregateOperator#and(Aggregations, int)}.
 *
 * @param agg The built-in aggregation function that is computed.
 * @param field The index of the Tuple field on which the aggregation function is applied.
 * @return An AggregateOperator that represents the aggregated DataSet.
 *
 * @see Tuple
 * @see Aggregations
 * @see AggregateOperator
 * @see DataSet
 */
public AggregateOperator<T> aggregate(Aggregations agg, int field) {
  return new AggregateOperator<>(this, agg, field, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Generic method to create an input DataSet with in {@link InputFormat}. The {@link DataSet} will not be
 * immediately created - instead, this method returns a {@link DataSet} that will be lazily created from
 * the input format once the program is executed.
 *
 * <p>The {@link DataSet} is typed to the given TypeInformation. This method is intended for input formats that
 * where the return type cannot be determined by reflection analysis, and that do not implement the
 * {@link ResultTypeQueryable} interface.
 *
 * @param inputFormat The input format used to create the data set.
 * @return A {@link DataSet} that represents the data created by the input format.
 *
 * @see #createInput(InputFormat)
 */
public <X> DataSource<X> createInput(InputFormat<X, ?> inputFormat, TypeInformation<X> producedType) {
  if (inputFormat == null) {
    throw new IllegalArgumentException("InputFormat must not be null.");
  }
  if (producedType == null) {
    throw new IllegalArgumentException("Produced type information must not be null.");
  }
  return new DataSource<>(this, inputFormat, producedType, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Initiates a Cross transformation.
 *
 * <p>A Cross transformation combines the elements of two
 *   {@link DataSet DataSets} into one DataSet. It builds all pair combinations of elements of
 *   both DataSets, i.e., it builds a Cartesian product.
 *
 *
 * <p>The resulting {@link org.apache.flink.api.java.operators.CrossOperator.DefaultCross} wraps each pair of crossed elements into a {@link Tuple2}, with
 * the element of the first input being the first field of the tuple and the element of the
 * second input being the second field of the tuple.
 *
 *
 * <p>Call {@link org.apache.flink.api.java.operators.CrossOperator.DefaultCross#with(org.apache.flink.api.common.functions.CrossFunction)} to define a
 * {@link org.apache.flink.api.common.functions.CrossFunction} which is called for
 * each pair of crossed elements. The CrossFunction returns a exactly one element for each pair of input elements.
 *
 * @param other The other DataSet with which this DataSet is crossed.
 * @return A DefaultCross that returns a Tuple2 for each pair of crossed elements.
 *
 * @see org.apache.flink.api.java.operators.CrossOperator.DefaultCross
 * @see org.apache.flink.api.common.functions.CrossFunction
 * @see DataSet
 * @see Tuple2
 */
public <R> CrossOperator.DefaultCross<T, R> cross(DataSet<R> other) {
  return new CrossOperator.DefaultCross<>(this, other, CrossHint.OPTIMIZER_CHOOSES, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Creates a new data set that contains a sequence of numbers. The data set will be created in parallel,
 * so there is no guarantee about the order of the elements.
 *
 * @param from The number to start at (inclusive).
 * @param to The number to stop at (inclusive).
 * @return A DataSet, containing all number in the {@code [from, to]} interval.
 */
public DataSource<Long> generateSequence(long from, long to) {
  return fromParallelCollection(new NumberSequenceIterator(from, to), BasicTypeInfo.LONG_TYPE_INFO, Utils.getCallLocationName());
}

代码示例来源:origin: apache/flink

/**
 * Applies a Reduce transformation on a non-grouped {@link DataSet}.
 *
 * <p>The transformation consecutively calls a {@link org.apache.flink.api.common.functions.RichReduceFunction}
 *   until only a single element remains which is the result of the transformation.
 * A ReduceFunction combines two elements into one new element of the same type.
 *
 * @param reducer The ReduceFunction that is applied on the DataSet.
 * @return A ReduceOperator that represents the reduced DataSet.
 *
 * @see org.apache.flink.api.common.functions.RichReduceFunction
 * @see ReduceOperator
 * @see DataSet
 */
public ReduceOperator<T> reduce(ReduceFunction<T> reducer) {
  if (reducer == null) {
    throw new NullPointerException("Reduce function must not be null.");
  }
  return new ReduceOperator<>(this, clean(reducer), Utils.getCallLocationName());
}

相关文章

微信公众号

最新文章

更多