1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.Reader;
7 import java.io.StringReader;
8
9 import org.apache.commons.io.IOUtils;
10
11 import net.sourceforge.pmd.lang.LanguageRegistry;
12 import net.sourceforge.pmd.lang.LanguageVersionHandler;
13 import net.sourceforge.pmd.lang.TokenManager;
14 import net.sourceforge.pmd.lang.jsp.JspLanguageModule;
15 import net.sourceforge.pmd.lang.jsp.ast.Token;
16 import net.sourceforge.pmd.util.IOUtil;
17
18 public class JSPTokenizer implements Tokenizer {
19
20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21 StringBuilder buffer = sourceCode.getCodeBuffer();
22 LanguageVersionHandler languageVersionHandler =
23 LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
24 Reader reader = null;
25
26 try {
27 reader = new StringReader(buffer.toString());
28 reader = IOUtil.skipBOM(reader);
29 TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
30 .getTokenManager(sourceCode.getFileName(), reader);
31 Token currentToken = (Token) tokenMgr.getNextToken();
32
33 while (currentToken.image.length() > 0) {
34 tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
35 currentToken.beginLine));
36 currentToken = (Token) tokenMgr.getNextToken();
37 }
38 } finally {
39 IOUtils.closeQuietly(reader);
40 }
41 tokenEntries.add(TokenEntry.getEOF());
42 }
43 }