org.apache.gobblin.configuration.State.getPropAsInt()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(9.2k)|赞(0)|评价(0)|浏览(97)

本文整理了Java中org.apache.gobblin.configuration.State.getPropAsInt()方法的一些代码示例,展示了State.getPropAsInt()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。State.getPropAsInt()方法的具体详情如下:
包路径:org.apache.gobblin.configuration.State
类名称:State
方法名:getPropAsInt

State.getPropAsInt介绍

[英]Get the value of a property as an integer.
[中]以整数形式获取属性的值。

代码示例

代码示例来源:origin: apache/incubator-gobblin

public RestApiConnector(State state) {
 this.state = state;
 this.authTokenTimeout =
   state.getPropAsInt(ConfigurationKeys.SOURCE_CONN_TIMEOUT, ConfigurationKeys.DEFAULT_CONN_TIMEOUT);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Get the number of {@link Dataset}s to be verified together. This allows multiple {@link Dataset}s
 * to share the same verification job, e.g., share the same query.
 */
private int getNumDatasetsVerifiedTogether() {
 return this.state.getPropAsInt(COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER,
   DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER);
}

代码示例来源:origin: apache/incubator-gobblin

public GenericJdbcBufferedInserter(State state, Connection conn) {
 super(state, conn);
 this.maxParamSize = state.getPropAsInt(WRITER_JDBC_MAX_PARAM_SIZE, DEFAULT_WRITER_JDBC_MAX_PARAM_SIZE);
}

代码示例来源:origin: apache/incubator-gobblin

private int getDataCompletenessVerificationThreadPoolSize() {
 return this.props.getPropAsInt(COMPACTION_COMPLETENESS_VERIFICATION_THREAD_POOL_SIZE,
   DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_THREAD_POOL_SIZE);
}

代码示例来源:origin: apache/incubator-gobblin

public MySqlBufferedInserter(State state, Connection conn) {
 super(state, conn);
 this.maxParamSize = state.getPropAsInt(WRITER_JDBC_MAX_PARAM_SIZE, DEFAULT_WRITER_JDBC_MAX_PARAM_SIZE);
}

代码示例来源:origin: apache/incubator-gobblin

public static int getPartitionIdFromState(State s) {
 return s.getPropAsInt(PARTITION_ID);
}

代码示例来源:origin: apache/incubator-gobblin

private int getThreadPoolSize() {
 return this.state.getPropAsInt(COMPACTION_THREAD_POOL_SIZE, DEFAULT_COMPACTION_THREAD_POOL_SIZE);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Constructor
 */
public PinotAuditCountHttpClient(State state) {
 int maxTotal = state.getPropAsInt(CONNECTION_MAX_TOTAL, DEFAULT_CONNECTION_MAX_TOTAL);
 int maxPerRoute = state.getPropAsInt(MAX_PER_ROUTE, DEFAULT_MAX_PER_ROUTE);
 cm = new PoolingHttpClientConnectionManager();
 cm.setMaxTotal(maxTotal);
 cm.setDefaultMaxPerRoute(maxPerRoute);
 httpClient = HttpClients.custom()
     .setConnectionManager(cm)
     .build();
 String host = state.getProp(TARGET_HOST);
 int port = state.getPropAsInt(TARGET_PORT);
 targetUrl = host + ":" + port + "/pql?pql=";
}

代码示例来源:origin: apache/incubator-gobblin

public HivePartitionVersionRetentionReaperPolicy(State state, HivePartitionDataset dataset) {
 super(state, dataset);
 Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS),
   "Missing required property " + ComplianceConfigurationKeys.REAPER_RETENTION_DAYS);
 this.retentionDays = state.getPropAsInt(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS);
}

代码示例来源:origin: apache/incubator-gobblin

private int getOwnFileCountThreshold (Dataset dataset) {
 int count = dataset.jobProps().getPropAsInt(MRCompactor.COMPACTION_LATEDATA_THRESHOLD_FILE_NUM,
   MRCompactor.DEFAULT_COMPACTION_LATEDATA_THRESHOLD_FILE_NUM);
 return count;
}

代码示例来源:origin: apache/incubator-gobblin

public RecoveryHelper(FileSystem fs, State state) throws IOException {
 this.fs = fs;
 this.persistDir = getPersistDir(state);
 this.retentionHours = state.getPropAsInt(PERSIST_RETENTION_KEY, DEFAULT_PERSIST_RETENTION);
}

代码示例来源:origin: apache/incubator-gobblin

public HivePartitionVersionRetentionCleanerPolicy(State state, HivePartitionDataset dataset) {
 super(state, dataset);
 Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS),
   "Missing required property " + ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS);
 Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS),
   "Missing required property " + ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS);
 Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS),
   "Missing required property " + ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS);
 this.backupRetentionVersions =
   this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS);
 this.backupRetentionDays = this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS);
 this.trashRetentionDays = this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS);
}

代码示例来源:origin: apache/incubator-gobblin

private int getMaxNumReducers() {
 return this.dataset.jobProps().getPropAsInt(COMPACTION_JOB_MAX_NUM_REDUCERS,
   DEFAULT_COMPACTION_JOB_MAX_NUM_REDUCERS);
}

代码示例来源:origin: apache/incubator-gobblin

GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, State wuState) throws IOException {
 _siteProperty = siteProperty;
 Preconditions.checkArgument(_siteProperty.endsWith("/"), "The site property must end in \"/\"");
 _client = client;
 _jobs = getHotStartJobs(wuState);
 API_REQUESTS_PER_SECOND = wuState.getPropAsDouble(GoogleWebMasterSource.KEY_PAGES_TUNING_REQUESTS_PER_SECOND, 4.5);
 PAGES_COUNT_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_COUNT_TUNING_COOLDOWN_TIME, 30);
 PAGES_GET_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_GET_TUNING_COOLDOWN_TIME, 5);
 LIMITER = new RateBasedLimiter(API_REQUESTS_PER_SECOND, TimeUnit.SECONDS);
 GET_PAGES_RETRIES = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_TUNING_MAX_RETRIES, 120);
}

代码示例来源:origin: apache/incubator-gobblin

@VisibleForTesting
GoogleDriveFsHelper(State state, Drive client, Closer closer) {
 this.closer = closer;
 this.fileSystem = this.closer.register(new GoogleDriveFileSystem(client));
 if (state.contains(BUFFER_SIZE_BYTE)) {
  this.bufferSizeByte = Optional.of(state.getPropAsInt(BUFFER_SIZE_BYTE));
 } else {
  this.bufferSizeByte = Optional.absent();
 }
}

代码示例来源:origin: apache/incubator-gobblin

public CachedInstanceKey(State state) {
 this.numOfEncryptionKeys = state.getPropAsInt(ConfigurationKeys.NUMBER_OF_ENCRYPT_KEYS, ConfigurationKeys.DEFAULT_NUMBER_OF_MASTER_PASSWORDS);
 this.useStrongEncryptor = shouldUseStrongEncryptor(state);
 this.fsURI = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC);
 this.masterPasswordFile = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC);
}

代码示例来源:origin: apache/incubator-gobblin

/**
  * @param state
  * @return Provides JdbcWriterCommands based on ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY
  */
 public JdbcWriterCommands newInstance(State state, Connection conn) {
  String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY,
    state.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1),
    state.getPropAsInt(ConfigurationKeys.FORK_BRANCH_ID_KEY, 0));
  String destType = state.getProp(destKey);
  Preconditions.checkNotNull(destType, destKey + " is required for underlying JDBC product name");
  return newInstance(Destination.of(DestinationType.valueOf(destType.toUpperCase()), state), conn);
 }
}

代码示例来源:origin: apache/incubator-gobblin

public BaseJdbcBufferedInserter(State state, Connection conn) {
 this.conn = conn;
 this.batchSize = state.getPropAsInt(WRITER_JDBC_INSERT_BATCH_SIZE, DEFAULT_WRITER_JDBC_INSERT_BATCH_SIZE);
 if (this.batchSize < 1) {
  throw new IllegalArgumentException(WRITER_JDBC_INSERT_BATCH_SIZE + " should be a positive number");
 }
 int maxWait = state.getPropAsInt(WRITER_JDBC_INSERT_RETRY_TIMEOUT, DEFAULT_WRITER_JDBC_INSERT_RETRY_TIMEOUT);
 int maxAttempts =
   state.getPropAsInt(WRITER_JDBC_INSERT_RETRY_MAX_ATTEMPT, DEFAULT_WRITER_JDBC_INSERT_RETRY_MAX_ATTEMPT);
 //retry after 2, 4, 8, 16... sec, allow at most maxWait sec delay
 this.retryer = RetryerBuilder.<Boolean> newBuilder().retryIfException()
   .withWaitStrategy(WaitStrategies.exponentialWait(1000, maxWait, TimeUnit.SECONDS))
   .withStopStrategy(StopStrategies.stopAfterAttempt(maxAttempts)).build();
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Calls {@link #getOptionallyThrottledFileSystem(FileSystem, int)} parsing the qps from the input {@link State}
 * at key {@link #MAX_FILESYSTEM_QPS}.
 * @throws IOException
 */
public static FileSystem getOptionallyThrottledFileSystem(FileSystem fs, State state) throws IOException {
 DeprecationUtils.renameDeprecatedKeys(state, MAX_FILESYSTEM_QPS, DEPRECATED_KEYS);
 if (state.contains(MAX_FILESYSTEM_QPS)) {
  return getOptionallyThrottledFileSystem(fs, state.getPropAsInt(MAX_FILESYSTEM_QPS));
 }
 return fs;
}

代码示例来源:origin: apache/incubator-gobblin

public HiveAvroSerDeManager(State props) throws IOException {
 super(props);
 if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) {
  this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), HadoopUtils.getConfFromState(props));
 } else {
  this.fs = FileSystem.get(HadoopUtils.getConfFromState(props));
 }
 this.useSchemaFile = props.getPropAsBoolean(USE_SCHEMA_FILE, DEFAULT_USE_SCHEMA_FILE);
 this.useSchemaTempFile = props.getPropAsBoolean(USE_SCHEMA_TEMP_FILE, DEFAULT_USE_SCHEMA_TEMP_FILE);
 this.schemaFileName = props.getProp(SCHEMA_FILE_NAME, DEFAULT_SCHEMA_FILE_NAME);
 this.schemaTempFileName = props.getProp(SCHEMA_TEMP_FILE_NAME, DEFAULT_SCHEMA_TEMP_FILE_NAME);
 this.schemaLiteralLengthLimit =
   props.getPropAsInt(SCHEMA_LITERAL_LENGTH_LIMIT, DEFAULT_SCHEMA_LITERAL_LENGTH_LIMIT);
 this.metricContext = Instrumented.getMetricContext(props, HiveAvroSerDeManager.class);
}

相关文章