com.sonar.sslr.api.Token.builder()方法的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(12.2k)|赞(0)|评价(0)|浏览(74)

本文整理了Java中com.sonar.sslr.api.Token.builder()方法的一些代码示例，展示了Token.builder()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Token.builder()方法的具体详情如下：
包路径：com.sonar.sslr.api.Token
类名称：Token
方法名：builder

Token.builder介绍

暂无

代码示例

代码示例来源：origin: org.codehaus.sonar.sslr/sslr-core

public void addToken(Token... tokens) {
 checkArgument(tokens.length > 0, "at least one token must be given");
 Token firstToken = tokens[0];
 Token firstTokenWithTrivia;
 // Performance optimization: no need to rebuild token, if there is no trivia
 if (trivia.isEmpty() && !firstToken.hasTrivia()) {
  firstTokenWithTrivia = firstToken;
 } else {
  firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build();
  trivia.clear();
 }
 this.tokens.add(firstTokenWithTrivia);
 if (tokens.length > 1) {
  this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length));
 }
}

代码示例来源：origin: SonarSource/sslr

public void addToken(Token... tokens) {
 if (tokens.length <= 0) {
  throw new IllegalArgumentException("at least one token must be given");
 }
 Token firstToken = tokens[0];
 Token firstTokenWithTrivia;
 // Performance optimization: no need to rebuild token, if there is no trivia
 if (trivia.isEmpty() && !firstToken.hasTrivia()) {
  firstTokenWithTrivia = firstToken;
 } else {
  firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build();
  trivia.clear();
 }
 this.tokens.add(firstTokenWithTrivia);
 if (tokens.length > 1) {
  this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length));
 }
}

代码示例来源：origin: org.sonarsource.sslr/sslr-core

public void addToken(Token... tokens) {
 if (tokens.length <= 0) {
  throw new IllegalArgumentException("at least one token must be given");
 }
 Token firstToken = tokens[0];
 Token firstTokenWithTrivia;
 // Performance optimization: no need to rebuild token, if there is no trivia
 if (trivia.isEmpty() && !firstToken.hasTrivia()) {
  firstTokenWithTrivia = firstToken;
 } else {
  firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build();
  trivia.clear();
 }
 this.tokens.add(firstTokenWithTrivia);
 if (tokens.length > 1) {
  this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length));
 }
}

代码示例来源：origin: org.sonarsource.sslr/sslr-testing-harness

public static Token.Builder mockTokenBuilder(TokenType type, String value) {
 try {
  return Token.builder()
    .setType(type)
    .setValueAndOriginalValue(value)
    .setURI(new URI("tests://unittest"))
    .setLine(1)
    .setColumn(1);
 } catch (URISyntaxException e) {
  throw new RuntimeException(e);
 }
}

代码示例来源：origin: SonarSource/sslr

public static Token.Builder mockTokenBuilder(TokenType type, String value) {
 try {
  return Token.builder()
    .setType(type)
    .setValueAndOriginalValue(value)
    .setURI(new URI("tests://unittest"))
    .setLine(1)
    .setColumn(1);
 } catch (URISyntaxException e) {
  throw new RuntimeException(e);
 }
}

代码示例来源：origin: org.codehaus.sonar-plugins.python/python-squid

@Override
public PreprocessorAction process(List<Token> tokens) {
 Token token = tokens.get(0);
 if (token.getType().equals(GenericTokenType.EOF)) {
  if (lexerState.indentationStack.isEmpty()) {
   return PreprocessorAction.NO_OPERATION;
  }
  List<Token> tokensToInject = Lists.newArrayList();
  while (lexerState.indentationStack.peek() > 0) {
   lexerState.indentationStack.pop();
   tokensToInject.add(Token.builder(token)
     .setURI(token.getURI())
     .setType(PythonTokenType.DEDENT)
     .setLine(token.getLine())
     .setColumn(token.getColumn())
     .setValueAndOriginalValue("")
     .build());
  }
  return new PreprocessorAction(0, Collections.<Trivia>emptyList(), tokensToInject);
 }
 return PreprocessorAction.NO_OPERATION;
}

代码示例来源：origin: org.sonarsource.python/python-squid

@Override
public PreprocessorAction process(List<Token> tokens) {
 Token token = tokens.get(0);
 if (token.getType().equals(GenericTokenType.EOF)) {
  if (lexerState.indentationStack.isEmpty()) {
   return PreprocessorAction.NO_OPERATION;
  }
  List<Token> tokensToInject = new ArrayList<>();
  while (lexerState.indentationStack.peek() > 0) {
   lexerState.indentationStack.pop();
   tokensToInject.add(Token.builder(token)
     .setURI(token.getURI())
     .setType(PythonTokenType.DEDENT)
     .setLine(token.getLine())
     .setColumn(token.getColumn())
     .setValueAndOriginalValue("")
     .build());
  }
  return new PreprocessorAction(0, Collections.<Trivia>emptyList(), tokensToInject);
 }
 return PreprocessorAction.NO_OPERATION;
}

代码示例来源：origin: SonarSource/sonar-python

@Override
public PreprocessorAction process(List<Token> tokens) {
 Token token = tokens.get(0);
 if (token.getType().equals(GenericTokenType.EOF)) {
  if (lexerState.indentationStack.isEmpty()) {
   return PreprocessorAction.NO_OPERATION;
  }
  List<Token> tokensToInject = new ArrayList<>();
  while (lexerState.indentationStack.peek() > 0) {
   lexerState.indentationStack.pop();
   tokensToInject.add(Token.builder(token)
     .setURI(token.getURI())
     .setType(PythonTokenType.DEDENT)
     .setLine(token.getLine())
     .setColumn(token.getColumn())
     .setValueAndOriginalValue("")
     .build());
  }
  return new PreprocessorAction(0, Collections.<Trivia>emptyList(), tokensToInject);
 }
 return PreprocessorAction.NO_OPERATION;
}

代码示例来源：origin: org.sonarsource.python/python-squid

private void processIndents(Lexer lexer, int line, int column, int indentationLevel) {
 if (indentationLevel > lexerState.indentationStack.peek()) {
  lexerState.indentationStack.push(indentationLevel);
  lexer.addToken(Token.builder()
    .setType(PythonTokenType.INDENT)
    .setValueAndOriginalValue(buffer.toString())
    .setURI(lexer.getURI())
    .setLine(line)
    .setColumn(column)
    .build());
 } else if (indentationLevel < lexerState.indentationStack.peek()) {
  while (indentationLevel < lexerState.indentationStack.peek()) {
   lexerState.indentationStack.pop();
   lexer.addToken(Token.builder()
     .setType(PythonTokenType.DEDENT)
     .setValueAndOriginalValue(buffer.toString())
     .setURI(lexer.getURI())
     .setLine(line)
     .setColumn(column)
     .build());
  }
 }
}

代码示例来源：origin: SonarSource/sonar-python

private void processIndents(Lexer lexer, int line, int column, int indentationLevel) {
 if (indentationLevel > lexerState.indentationStack.peek()) {
  lexerState.indentationStack.push(indentationLevel);
  lexer.addToken(Token.builder()
    .setType(PythonTokenType.INDENT)
    .setValueAndOriginalValue(buffer.toString())
    .setURI(lexer.getURI())
    .setLine(line)
    .setColumn(column)
    .build());
 } else if (indentationLevel < lexerState.indentationStack.peek()) {
  while (indentationLevel < lexerState.indentationStack.peek()) {
   lexerState.indentationStack.pop();
   lexer.addToken(Token.builder()
     .setType(PythonTokenType.DEDENT)
     .setValueAndOriginalValue(buffer.toString())
     .setURI(lexer.getURI())
     .setLine(line)
     .setColumn(column)
     .build());
  }
 }
}

代码示例来源：origin: org.codehaus.sonar-plugins.python/python-squid

private void processIndents(Lexer lexer, int line, int column, int indentationLevel) {
 if (indentationLevel > lexerState.indentationStack.peek()) {
  lexerState.indentationStack.push(indentationLevel);
  lexer.addToken(Token.builder()
    .setType(PythonTokenType.INDENT)
    .setValueAndOriginalValue(buffer.toString())
    .setURI(lexer.getURI())
    .setLine(line)
    .setColumn(column)
    .build());
 } else if (indentationLevel < lexerState.indentationStack.peek()) {
  while (indentationLevel < lexerState.indentationStack.peek()) {
   lexerState.indentationStack.pop();
   lexer.addToken(Token.builder()
     .setType(PythonTokenType.DEDENT)
     .setValueAndOriginalValue(buffer.toString())
     .setURI(lexer.getURI())
     .setLine(line)
     .setColumn(column)
     .build());
  }
 }
}

代码示例来源：origin: sonar-perl/sonar-perl

@Override
public boolean consume(CodeReader code, Lexer output) {
  int line = code.getLinePosition();
  int column = code.getColumnPosition();
  index = 0;
  ch = code.charAt(index);
  if ((ch != '\'') && (ch != '\"') && (ch != '`')) {
    return false;
  }
  if (!read(code)) {
    return false;
  }
  for (int i = 0; i < index; i++) {
    sb.append((char) code.pop());
  }
  output.addToken(Token.builder()
      .setLine(line)
      .setColumn(column)
      .setURI(output.getURI())
      .setValueAndOriginalValue(sb.toString())
      .setType(PerlTokenType.STRING)
      .build());
  sb.setLength(0);
  return true;
}

代码示例来源：origin: sonar-perl/sonar-perl

private boolean processNewLine(CodeReader code, Lexer output) {
  if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PerlTokenType.NEWLINE))) {
    // Blank line
    consumeEOL(code);
    return true;
  }
  // NEWLINE token
  output.addToken(Token.builder()
      .setLine(code.getLinePosition())
      .setColumn(code.getColumnPosition())
      .setURI(output.getURI())
      .setType(PerlTokenType.NEWLINE)
      .setValueAndOriginalValue("\n")
      .setGeneratedCode(true)
      .build());
  consumeEOL(code);
  return false;
}

代码示例来源：origin: SonarSource/sonar-python

private boolean processNewLine(CodeReader code, Lexer output) {
 if (isImplicitLineJoining()) {
  // Implicit line joining
  joinLines(code);
  return true;
 }
 if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PythonTokenType.NEWLINE))) {
  // Blank line
  consumeEOL(code);
  return true;
 }
 // NEWLINE token
 output.addToken(Token.builder()
   .setLine(code.getLinePosition())
   .setColumn(code.getColumnPosition())
   .setURI(output.getURI())
   .setType(PythonTokenType.NEWLINE)
   .setValueAndOriginalValue("\n")
   .setGeneratedCode(true)
   .build());
 consumeEOL(code);
 return false;
}

代码示例来源：origin: sonar-perl/sonar-perl

@Override
public boolean consume(CodeReader code, Lexer output) {
  int line = code.getLinePosition();
  int column = code.getColumnPosition();
  sb = new StringBuilder();
  if (column == 0 && code.popTo(POD_START, sb) > 0) {
    String pod = consumePod(code);
    // POD token
    output.addToken(Token.builder()
      .setLine(line)
      .setColumn(column)
      .setURI(output.getURI())
      .setType(GenericTokenType.COMMENT)
      .setValueAndOriginalValue(pod)
      .setGeneratedCode(true)
      .build());
    return true;
  }
  return false;
}

代码示例来源：origin: org.sonarsource.sslr/sslr-core

private List<Token> lex(Reader reader) {
 tokens = new ArrayList<>();
 initPreprocessors();
 CodeReader code = new CodeReader(reader, configuration);
 try {
  channelDispatcher.consume(code, this);
  addToken(Token.builder()
   .setType(EOF)
   .setValueAndOriginalValue("EOF")
   .setURI(uri)
   .setLine(code.getLinePosition())
   .setColumn(code.getColumnPosition())
   .build());
  preprocess();
  return getTokens();
 } catch (Exception e) {
  throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : "
   + code.getColumnPosition() + " in file : " + uri, e);
 }
}

代码示例来源：origin: SonarSource/sslr

private List<Token> lex(Reader reader) {
 tokens = new ArrayList<>();
 initPreprocessors();
 CodeReader code = new CodeReader(reader, configuration);
 try {
  channelDispatcher.consume(code, this);
  addToken(Token.builder()
   .setType(EOF)
   .setValueAndOriginalValue("EOF")
   .setURI(uri)
   .setLine(code.getLinePosition())
   .setColumn(code.getColumnPosition())
   .build());
  preprocess();
  return getTokens();
 } catch (Exception e) {
  throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : "
   + code.getColumnPosition() + " in file : " + uri, e);
 }
}

代码示例来源：origin: org.codehaus.sonar.sslr/sslr-core

private List<Token> lex(Reader reader) {
 tokens = Lists.newArrayList();
 initPreprocessors();
 CodeReader code = new CodeReader(reader, configuration);
 try {
  channelDispatcher.consume(code, this);
  addToken(Token.builder()
   .setType(EOF)
   .setValueAndOriginalValue("EOF")
   .setURI(uri)
   .setLine(code.getLinePosition())
   .setColumn(code.getColumnPosition())
   .build());
  preprocess();
  return getTokens();
 } catch (Exception e) {
  throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : "
   + code.getColumnPosition() + " in file : " + uri, e);
 }
}

代码示例来源：origin: org.sonarsource.sslr/sslr-core

@Override
public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) {
 int[] lineAndColumn = input.lineAndColumnAt(startIndex);
 Token token = Token.builder()
  .setType(type == null ? UNDEFINED_TOKEN_TYPE : type)
  .setLine(lineAndColumn[0])
  .setColumn(lineAndColumn[1] - 1)
  .setValueAndOriginalValue(input.substring(startIndex, endIndex))
  .setURI(input.uri())
  .setGeneratedCode(false)
  .setTrivia(trivias)
  .build();
 AstNode astNode = new AstNode(token);
 astNode.setFromIndex(startIndex);
 astNode.setToIndex(endIndex);
 return astNode;
}

代码示例来源：origin: SonarSource/sslr

@Override
public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) {
 int[] lineAndColumn = input.lineAndColumnAt(startIndex);
 Token token = Token.builder()
  .setType(type == null ? UNDEFINED_TOKEN_TYPE : type)
  .setLine(lineAndColumn[0])
  .setColumn(lineAndColumn[1] - 1)
  .setValueAndOriginalValue(input.substring(startIndex, endIndex))
  .setURI(input.uri())
  .setGeneratedCode(false)
  .setTrivia(trivias)
  .build();
 AstNode astNode = new AstNode(token);
 astNode.setFromIndex(startIndex);
 astNode.setToIndex(endIndex);
 return astNode;
}

内容来源于网络，如有侵权，请联系作者删除！

token

相关文章

    查看更多
    微信公众号

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com