1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.IOException;
7
8 public interface Tokenizer {
9 String IGNORE_LITERALS = "ignore_literals";
10 String IGNORE_IDENTIFIERS = "ignore_identifiers";
11 String IGNORE_ANNOTATIONS = "ignore_annotations";
12
13
14
15
16
17 String IGNORE_USINGS = "ignore_usings";
18
19
20
21
22
23
24
25 String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks";
26
27
28
29
30
31
32
33 String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern";
34
35 String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif";
36
37 void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException;
38 }