de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token.<init>()方法的使用及代码示例
本文整理了Java中de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token.<init>()方法的一些代码示例，展示了Token.<init>()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Token.<init>()方法的具体详情如下：
包路径：de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token
类名称：Token
方法名：<init>
Token.<init>介绍
[英]Never called. Disable default constructor
[中]从没打过电话。禁用默认构造函数
代码示例
代码示例来源：origin: de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.tokit-asl
private Token addToken(JCas aJCas, int offset, String text, int start, int end,
List<Token> toAdd)
{
// No adding empty tokens
if (end == start) {
return null;
}
Token t = new Token(aJCas, offset + start, offset + end);
toAdd.add(t);
return t;
}代码示例来源：origin: de.unidue.ltl.flextag/flextag-features
public Token get(JCas jcas, TextClassificationTarget target)
{
Integer integer = super.unitBegin2Idx.get(target.getBegin());
if (integer + shiftIdx < 0) {
return new Token(jcas, -1, -1);
}
if (integer + shiftIdx >= unitBegin2Idx.size()) {
return new Token(jcas, -1, -1);
}
return units.get(integer + shiftIdx);
}代码示例来源：origin: dkpro/dkpro-tc
protected void setToken(JCas aJCas, int begin, int end)
{
Token token = new Token(aJCas, begin, end);
token.addToIndexes();
}代码示例来源：origin: de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.tokit-asl
/**
* Create tokens using the boundary pattern defined in {@link #whitespace}.
*
* @param aJCas
* the {@link JCas}
* @param text
* the text
*/
private void createTokens(JCas aJCas, String text)
{
Matcher whitespaceMatcher = whitespace.matcher(text);
int previousStart = 0;
while (whitespaceMatcher.find()) {
int end = whitespaceMatcher.start();
Token token = new Token(aJCas, previousStart, end);
token.addToIndexes(aJCas);
previousStart = whitespaceMatcher.end();
}
}
}代码示例来源：origin: de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.tokit-asl
/**
* Create tokens using the boundary pattern defined in {@link #whitespace}.
*
* @param aJCas
* the {@link JCas}
* @param text
* the text
* @param zoneBegin
*/
private void createTokens(JCas aJCas, String text, int zoneBegin)
{
Matcher tokenBoundaryMatcher = tokenBoundaryPattern.matcher(text);
int previousStart = 0;
while (tokenBoundaryMatcher.find()) {
int end = tokenBoundaryMatcher.start();
Token token = new Token(aJCas, zoneBegin + previousStart, zoneBegin + end);
token.addToIndexes(aJCas);
previousStart = tokenBoundaryMatcher.end();
}
}
}代码示例来源：origin: de.tudarmstadt.ukp.clarin.webanno/webanno-tsv
private AnnotationUnit createTokens(JCas aJCas, String[] lines, int begin, int end) {
// subtokens should not be consider as tokens. example 1-2.1 ==> subtoken under token 2
if (!lines[0].contains(".")) {
Token token = new Token(aJCas, begin, end);
AnnotationUnit unit = new AnnotationUnit(begin, end, false, "");
units.add(unit);
token.addToIndexes();
token2Units.put(lines[0], unit);
units2Tokens.put(unit, token);
return unit;
} else {
AnnotationUnit unit = new AnnotationUnit(begin, end, true, "");
units.add(unit);
token2Units.put(lines[0], unit);
return unit;
}
}代码示例来源：origin: de.tudarmstadt.ukp.clarin.webanno/webanno-io-tsv
private AnnotationUnit createTokens(JCas aJCas, String[] lines, int begin, int end)
{
// subtokens should not be consider as tokens. example 1-2.1 ==> subtoken under token 2
if (!lines[0].contains(".")) {
Token token = new Token(aJCas, begin, end);
AnnotationUnit unit = new AnnotationUnit(begin, end, false, "");
units.add(unit);
token.addToIndexes();
token2Units.put(lines[0], unit);
units2Tokens.put(unit, token);
return unit;
}
else {
AnnotationUnit unit = new AnnotationUnit(begin, end, true, "");
units.add(unit);
token2Units.put(lines[0], unit);
return unit;
}
}代码示例来源：origin: webanno/webanno
private AnnotationUnit createTokens(JCas aJCas, String[] lines, int begin, int end)
{
// subtokens should not be consider as tokens. example 1-2.1 ==> subtoken under token 2
if (!lines[0].contains(".")) {
Token token = new Token(aJCas, begin, end);
AnnotationUnit unit = new AnnotationUnit(begin, end, false, "");
units.add(unit);
token.addToIndexes();
token2Units.put(lines[0], unit);
units2Tokens.put(unit, token);
return unit;
}
else {
AnnotationUnit unit = new AnnotationUnit(begin, end, true, "");
units.add(unit);
token2Units.put(lines[0], unit);
return unit;
}
}代码示例来源：origin: hltfbk/Excitement-Open-Platform
private Annotation getAnnotationObject(JCas aJCas, String viewName, String type, Integer[] index) throws Exception{
if (type.matches(txpAnn.token))
return new Token(aJCas.getView(viewName));
if (type.matches(txpAnn.lemma))
return new Lemma(aJCas.getView(viewName));
if (type.matches(txpAnn.pos))
return getAnnotationObject(aJCas, viewName, txpAnn.getAnnotation(type).get(index), POS.class);
if (type.matches(txpAnn.sentence))
return new Sentence(aJCas.getView(viewName));
if (type.matches(txpAnn.ne))
return getAnnotationObject(aJCas, viewName, txpAnn.getNEtype(index), NamedEntity.class);
return new Annotation(aJCas.getView(viewName));
}代码示例来源：origin: UKPLab/argument-reasoning-comprehension-task
private static void copyParagraphAndTokenAnnotations(JCas source, JCas target)
{
if (!source.getDocumentText().equals(target.getDocumentText())) {
throw new IllegalArgumentException("Source and target have different content");
}
for (Paragraph p : JCasUtil.select(source, Paragraph.class)) {
Paragraph paragraph = new Paragraph(target);
paragraph.setBegin(p.getBegin());
paragraph.setEnd(p.getEnd());
paragraph.addToIndexes();
}
for (Token t : JCasUtil.select(source, Token.class)) {
Token token = new Token(target);
token.setBegin(t.getBegin());
token.setEnd(t.getEnd());
token.addToIndexes();
}
}代码示例来源：origin: de.tudarmstadt.ukp.dkpro.keyphrases/de.tudarmstadt.ukp.dkpro.keyphrases.decompounding-asl
@Override
public void process(final JCas aJCas)
throws AnalysisEngineProcessException
{
Token token;
for (Compound compound : JCasUtil.select(aJCas, Compound.class)) {
final Token compoundToken = JCasUtil.selectCovered(aJCas, Token.class,
compound.getBegin(), compound.getEnd()).get(0);
for (Split compoundPart : compound.getSplitsWithoutMorpheme(compoundSplitLevel)) {
token = new Token(aJCas);
token.setBegin(compoundPart.getBegin());
token.setEnd(compoundPart.getEnd());
token.setPos(compoundToken.getPos());
token.addToIndexes();
}
}
}代码示例来源：origin: dkpro/dkpro-core
protected Token createToken(final JCas aJCas, final String aForm, final int aBegin,
final int aEnd)
{
int[] span = new int[] { aBegin, aEnd };
trim(aJCas.getDocumentText(), span);
if (!isEmpty(span[0], span[1]) && isWriteToken()) {
Token seg = new Token(aJCas, span[0], span[1]);
if (aForm != null && writeForm) {
seg.setText(aForm);
}
seg.addToIndexes(aJCas);
return seg;
}
else {
return null;
}
}代码示例来源：origin: de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.api.segmentation-asl
protected Token createToken(final JCas aJCas, final String aForm, final int aBegin,
final int aEnd)
{
int[] span = new int[] { aBegin, aEnd };
trim(aJCas.getDocumentText(), span);
if (!isEmpty(span[0], span[1]) && isWriteToken()) {
Token seg = new Token(aJCas, span[0], span[1]);
if (aForm != null && writeForm) {
seg.setText(aForm);
}
seg.addToIndexes(aJCas);
return seg;
}
else {
return null;
}
}代码示例来源：origin: de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.io.penntree-asl
private void annotateTokenWithTag(JCas aJCas, String aToken, String aTag, int aCurrPosInText)
{
if (readToken) {
// Token
Token token = new Token(aJCas, aCurrPosInText, aToken.length() + aCurrPosInText);
token.addToIndexes();
if (readPOS) {
// Tag
Type posTag = posMappingProvider.getTagType(aTag);
POS pos = (POS) aJCas.getCas().createAnnotation(posTag, token.getBegin(),
token.getEnd());
pos.setPosValue(aTag);
POSUtils.assignCoarseValue(pos);
pos.addToIndexes();
// Set the POS for the Token
token.setPos(pos);
}
}
}
}代码示例来源：origin: de.tudarmstadt.ukp.clarin.webanno/webanno-api-dao
public static void tokenize(JCas aJCas)
{
BreakIterator bi = BreakIterator.getWordInstance(Locale.US);
for (Sentence s : select(aJCas, Sentence.class)) {
bi.setText(s.getCoveredText());
int last = bi.first();
int cur = bi.next();
while (cur != BreakIterator.DONE) {
int[] span = new int[] { last, cur };
trim(s.getCoveredText(), span);
if (!isEmpty(span[0], span[1])) {
Token seg = new Token(aJCas, span[0] + s.getBegin(), span[1] + s.getBegin());
seg.addToIndexes(aJCas);
}
last = cur;
cur = bi.next();
}
}
}代码示例来源：origin: webanno/webanno
public static void tokenize(JCas aJCas)
{
BreakIterator bi = BreakIterator.getWordInstance(Locale.US);
for (Sentence s : select(aJCas, Sentence.class)) {
bi.setText(s.getCoveredText());
int last = bi.first();
int cur = bi.next();
while (cur != BreakIterator.DONE) {
int[] span = new int[] { last, cur };
trim(s.getCoveredText(), span);
if (!isEmpty(span[0], span[1])) {
Token seg = new Token(aJCas, span[0] + s.getBegin(), span[1] + s.getBegin());
seg.addToIndexes(aJCas);
}
last = cur;
cur = bi.next();
}
}
}代码示例来源：origin: webanno/webanno
@Test
public void testZeroLengthSpanBetweenAdjacentTokens() throws Exception
{
JCas jcas = makeJCas();
jcas.setDocumentText("word.");
new Token(jcas, 0,4).addToIndexes();
new Token(jcas, 4,5).addToIndexes();
new Sentence(jcas, 0,5).addToIndexes();
CAS cas = jcas.getCas();
Type simpleSpanType = cas.getTypeSystem().getType("webanno.custom.SimpleSpan");
// Insert zero-width annotation between the adjacent tokens (at end of first token).
AnnotationFS fs1a = cas.createAnnotation(simpleSpanType, 4, 4);
cas.addFsToIndexes(fs1a);
writeAndAssertEquals(jcas,
WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList("webanno.custom.SimpleSpan"));
}代码示例来源：origin: webanno/webanno
@Test
public void testTwoSentencesWithNoSpaceInBetween() throws Exception
{
TypeSystemDescription global = TypeSystemDescriptionFactory.createTypeSystemDescription();
TypeSystemDescription local = TypeSystemDescriptionFactory
.createTypeSystemDescriptionFromPath(
"src/test/resources/desc/type/webannoTestTypes.xml");
TypeSystemDescription merged = CasCreationUtils.mergeTypeSystems(asList(global, local));
JCas jcas = JCasFactory.createJCas(merged);
DocumentMetaData.create(jcas).setDocumentId("doc");
jcas.setDocumentText("onetwo");
new Token(jcas, 0, 3).addToIndexes();
new Sentence(jcas, 0, 3).addToIndexes();
new Token(jcas, 3, 6).addToIndexes();
new Sentence(jcas, 3, 6).addToIndexes();
writeAndAssertEquals(jcas);
}代码示例来源：origin: webanno/webanno
public static FeatureStructure makeLinkFS(JCas aJCas, String aSlotLabel, int aTargetBegin,
int aTargetEnd)
{
Token token1 = new Token(aJCas, aTargetBegin, aTargetEnd);
token1.addToIndexes();
Type linkType = aJCas.getTypeSystem().getType(LINK_TYPE);
FeatureStructure linkA1 = aJCas.getCas().createFS(linkType);
linkA1.setStringValue(linkType.getFeatureByBaseName("role"), aSlotLabel);
linkA1.setFeatureValue(linkType.getFeatureByBaseName("target"), token1);
aJCas.getCas().addFsToIndexes(linkA1);
return linkA1;
}
}代码示例来源：origin: webanno/webanno
public static FeatureStructure makeLinkFS(JCas aJCas, String aSlotLabel, int aTargetBegin,
int aTargetEnd)
{
Token token1 = new Token(aJCas, aTargetBegin, aTargetEnd);
token1.addToIndexes();
Type linkType = aJCas.getTypeSystem().getType(LINK_TYPE);
FeatureStructure linkA1 = aJCas.getCas().createFS(linkType);
linkA1.setStringValue(linkType.getFeatureByBaseName("role"), aSlotLabel);
linkA1.setFeatureValue(linkType.getFeatureByBaseName("target"), token1);
aJCas.getCas().addFsToIndexes(linkA1);
return linkA1;
}
}内容来源于网络，如有侵权，请联系作者删除！
