org.apache.gobblin.configuration.State.setProp()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(8.2k)|赞(0)|评价(0)|浏览(109)

本文整理了Java中org.apache.gobblin.configuration.State.setProp()方法的一些代码示例,展示了State.setProp()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。State.setProp()方法的具体详情如下:
包路径:org.apache.gobblin.configuration.State
类名称:State
方法名:setProp

State.setProp介绍

[英]Set a property.

Both key and value are stored as strings.
[中]设置一个属性。
键和值都存储为字符串。

代码示例

代码示例来源:origin: apache/incubator-gobblin

/**
 * Set number of schema evolution DDLs as Sla event metadata
 */
public static void setEvolutionMetadata(State state, List<String> evolutionDDLs) {
 state.setProp(EventConstants.SCHEMA_EVOLUTION_DDLS_NUM, evolutionDDLs == null ? 0 : evolutionDDLs.size());
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Disable Hive watermarker. This is necessary when there is no concrete source table where watermark can be inferred.
 */
public static void disableHiveWatermarker(State state) {
 state.setProp(USE_WATERMARKER_KEY, Boolean.toString(false));
}

代码示例来源:origin: apache/incubator-gobblin

/***
 * Serialize a {@link QueryBasedHivePublishEntity} into a {@link State} at {@link #SERIALIZED_PUBLISH_TABLE_COMMANDS}.
 * @param state {@link State} to serialize entity into.
 * @param queryBasedHivePublishEntity to carry to publisher.
 */
public static void serializePublishCommands(State state, QueryBasedHivePublishEntity queryBasedHivePublishEntity) {
 state.setProp(HiveAvroORCQueryGenerator.SERIALIZED_PUBLISH_TABLE_COMMANDS,
   GSON.toJson(queryBasedHivePublishEntity));
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Given a {@link String} in octal notation, set a key, value pair in the given {@link State} for the writer to
 * use when creating files. This method should be used in conjunction with {@link #deserializeWriterFilePermissions(State, int, int)}.
 */
public static void setWriterFileOctalPermissions(State state, int numBranches, int branchId,
  String octalPermissions) {
 state.setProp(
   ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId),
   octalPermissions);
}

代码示例来源:origin: apache/incubator-gobblin

public static State getStateFromConf(Configuration conf) {
 State state = new State();
 for (Entry<String, String> entry : conf) {
  state.setProp(entry.getKey(), entry.getValue());
 }
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Add the input file to the Hive session before running the task.
 */
public static void addFile(State state, String file) {
 state.setProp(ADD_FILES, state.getProp(ADD_FILES, "") + "," + file);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Add the input jar to the Hive session before running the task.
 */
public static void addJar(State state, String jar) {
 state.setProp(ADD_JARS, state.getProp(ADD_JARS, "") + "," + jar);
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 state.setProp("RecordsWritten", recordsWritten());
 try {
  state.setProp("BytesWritten", bytesWritten());
 } catch (Exception exception) {
  // If Writer fails to return bytesWritten, it might not be implemented, or implemented incorrectly.
  // Omit property instead of failing.
 }
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

public static FileSystem getOwnerFs(State state, Optional<String> owner)
  throws IOException {
 if (owner.isPresent()) {
  state.setProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_PROXY_USER, owner.get());
 }
 ProxyUtils.setProxySettingsForFs(state);
 return WriterUtils.getWriterFs(state);
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Write record count to a specific directory.
 * File name is {@link InputRecordCountHelper#RECORD_COUNT_FILE}
 * @param fs file system in use
 * @param dir directory where a record file is located
 */
@Deprecated
public static void writeRecordCount (FileSystem fs, Path dir, long count) throws IOException {
  State state = loadState(fs, dir);
  state.setProp(CompactionSlaEventHelper.RECORD_COUNT_TOTAL, count);
  saveState(fs, dir, state);
}

代码示例来源:origin: apache/incubator-gobblin

private State getTestState(long recordsRead, long recordsWritten, double range) {
  State state = new State();
  state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, recordsRead);
  state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, recordsWritten);
  state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, range);
  return state;
 }
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 if (this.writer instanceof FinalState) {
  state.addAll(((FinalState)this.writer).getFinalState());
 } else {
  LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
 }
 state.setProp(THROTTLED_TIME_KEY, this.throttledTime);
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

@Override
public State getFinalState() {
 State state = new State();
 if (this.writer instanceof FinalState) {
  state.addAll(((FinalState)this.writer).getFinalState());
 } else {
  LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
 }
 state.setProp(FAILED_WRITES_KEY, this.failedWrites);
 return state;
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Create a CheckpointableWatermarkState object from a CheckpointableWatermark
 * @param watermark: the checkpointable watermark
 * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}.
 */
public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) {
 super.setProp(watermark.getSource(), gson.toJsonTree(watermark));
 super.setId(watermark.getSource());
}

代码示例来源:origin: apache/incubator-gobblin

/**
 * Set the average time in milliseconds to pull a record of a partition, which will be stored in property
 * "[topicname].[partitionid].avg.record.millis".
 */
public static void setPartitionAvgRecordMillis(State state, KafkaPartition partition, double millis) {
 state.setProp(
   getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_MILLIS,
   millis);
}

代码示例来源:origin: apache/incubator-gobblin

@Test
public void testPolicyChecker()
  throws Exception {
 State state = new State();
 state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy");
 state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL");
 TaskLevelPolicyCheckResults results = getPolicyResults(state);
 for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
  Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
 }
}

代码示例来源:origin: apache/incubator-gobblin

@Test
public void testMultiplePolicies()
  throws Exception {
 State state = new State();
 state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST,
   "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy,org.apache.gobblin.qualitychecker.TestTaskLevelPolicy");
 state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL");
 TaskLevelPolicyCheckResults results = getPolicyResults(state);
 for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
  Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
 }
}

代码示例来源:origin: apache/incubator-gobblin

@Test public void testGetPersistDir() throws Exception {
 State state = new State();
 Assert.assertFalse(RecoveryHelper.getPersistDir(state).isPresent());
 state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath());
 Assert.assertTrue(RecoveryHelper.getPersistDir(state).isPresent());
 Assert.assertTrue(RecoveryHelper.getPersistDir(state).get().toUri().getPath().
   startsWith(this.tmpDir.getAbsolutePath()));
}

代码示例来源:origin: apache/incubator-gobblin

public void userCreatedStagingTable() throws SQLException {
 this.state.setProp(ConfigurationKeys.WRITER_STAGING_TABLE, STAGING_TABLE);
 when(this.commands.isEmpty(DB, STAGING_TABLE)).thenReturn(Boolean.TRUE);
 this.initializer.initialize();
 Assert.assertEquals(STAGING_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE));
 verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString());
 verify(this.commands, never()).truncate(anyString(), anyString());
 verify(this.commands, never()).drop(anyString(), anyString());
}

代码示例来源:origin: apache/incubator-gobblin

@Test
public void testConfigureFromState() {
 HttpClientConfiguratorLoader loader = new HttpClientConfiguratorLoader(new State());
 Assert.assertEquals(loader.getConfigurator().getClass(), DefaultHttpClientConfigurator.class);
 State state = new State();
 state.setProp(HttpClientConfiguratorLoader.HTTP_CLIENT_CONFIGURATOR_TYPE_FULL_KEY, "default");
 loader = new HttpClientConfiguratorLoader(state);
 Assert.assertEquals(loader.getConfigurator().getClass(), DefaultHttpClientConfigurator.class);
}

相关文章