1 package emissary.util;
2
3 import emissary.test.core.junit5.UnitTest;
4
5 import org.junit.jupiter.api.Test;
6
7 import java.nio.charset.StandardCharsets;
8
9 import static org.junit.jupiter.api.Assertions.assertEquals;
10 import static org.junit.jupiter.api.Assertions.assertNull;
11 import static org.junit.jupiter.api.Assertions.assertTrue;
12
13 class LineTokenizerTest extends UnitTest {
14
15 private static final String W = "Президент Буш";
16
17 @Test
18 void testCharset() {
19 LineTokenizer lt = new LineTokenizer((W + "\n").getBytes(), StandardCharsets.UTF_8);
20 assertEquals(W, lt.nextToken(), "UTF-8 passed through clean");
21 }
22
23 @Test
24 void testStringCharset() {
25 LineTokenizer lt = new LineTokenizer((W + "\n").getBytes(), StandardCharsets.UTF_8);
26 assertEquals(W, lt.nextToken(), "UTF-8 passed through clean");
27 }
28
29 @Test
30 void testParsing() {
31 LineTokenizer lt = new LineTokenizer("ABC\nDEF\r\nGHI\n\nJKL\r\n\r\nMNO".getBytes());
32 assertEquals(7, lt.countTokens(), "Line counting");
33 assertTrue(lt.hasMoreTokens(), "Token setup");
34 assertEquals(0, lt.index, "Index should be at the beginning of the string");
35 assertEquals("ABC", lt.nextToken(), "Line parsing with LF");
36 assertEquals(4, lt.index, "Index should be at the index of 'D'");
37 assertEquals("DEF\r", lt.nextToken(), "Line parsing with CRLF");
38 assertEquals(9, lt.index, "Index should be at the index of 'G'");
39 assertEquals("GHI", lt.nextToken(), "Line parsing before double LF");
40 assertEquals(13, lt.index, "Index should be at the index of the '\n' before 'JKL'");
41 assertEquals("", lt.nextToken(), "Blank line with LF");
42 assertEquals(14, lt.index, "Index should be at the index of 'J'");
43 assertEquals("JKL\r", lt.nextToken(), "Line pargin before double CRLF");
44 assertEquals(19, lt.index, "Index should be at the index of '\r' before '\nMNO'");
45 assertEquals("\r", lt.nextToken(), "Blank line with CRLF");
46 assertEquals(21, lt.index, "Index should at the index of 'M'");
47 assertEquals("MNO", lt.nextToken(), "Trailing portion without LF");
48 assertEquals(24, lt.index, "Index should be at the end of the string");
49 assertNull(lt.nextToken(), "No remaining tokens");
50 assertEquals(24, lt.index, "Index should not have changed");
51 }
52
53 @Test
54 void testParsingWithSpecifiedDelimiter() {
55 LineTokenizer lt = new LineTokenizer("ABC\r\nDEF\r\nGHI\n\nJKL\r\n\r\nMNO".getBytes(), (byte) '\r', StandardCharsets.UTF_8);
56 assertEquals(5, lt.countTokens(), "Line counting");
57 assertTrue(lt.hasMoreTokens(), "Token setup");
58 assertEquals("ABC", lt.nextToken(), "Line parsing with LF");
59 assertEquals("\nDEF", lt.nextToken(), "Line parsing with CRLF");
60 }
61
62 @Test
63 void testParsingWithSpecifiedDelimiterAndStringCharset() {
64 LineTokenizer lt = new LineTokenizer("ABC\r\nDEF\r\nGHI\n\nJKL\r\n\r\nMNO".getBytes(), (byte) '\r', "UTF-8");
65 assertEquals(5, lt.countTokens(), "Line counting");
66 assertTrue(lt.hasMoreTokens(), "Token setup");
67 assertEquals("ABC", lt.nextToken(), "Line parsing with LF");
68 assertEquals("\nDEF", lt.nextToken(), "Line parsing with CRLF");
69 }
70
71 @Test
72 void testPushback() {
73 LineTokenizer lt = new LineTokenizer("ABC\nDEF\nGHI\nJKL\n".getBytes());
74 assertEquals(4, lt.countTokens(), "Line counting");
75 assertEquals("ABC", lt.nextToken(), "First token");
76 assertEquals(4, lt.index, "Index should be at the index of 'D'");
77 lt.pushBack();
78 assertEquals(0, lt.index, "Index should be at the start of the string");
79 assertEquals(4, lt.countTokens(), "Line counting after push");
80 assertEquals("ABC", lt.nextToken(), "First token after push");
81 assertEquals(4, lt.index, "Index should be at the index of 'D'");
82
83 assertEquals("DEF", lt.nextToken(), "Second token");
84 assertEquals(8, lt.index, "Index should be at the index of 'G'");
85 lt.pushBack();
86 assertEquals(4, lt.index, "Index should be at the index of 'D'");
87 assertEquals("DEF", lt.nextToken(), "Second token after push");
88 assertEquals(8, lt.index, "Index should be at the index of 'G'");
89 lt.pushBack();
90 assertEquals(4, lt.index, "Index should be at the index of 'D'");
91 assertEquals("DEF", lt.nextToken(), "Second token after push");
92 assertEquals(8, lt.index, "Index should be at the index of 'G'");
93
94 lt.nextToken();
95 assertEquals("JKL", lt.nextToken(), "End of the string");
96 assertEquals(15, lt.index, "Index should be at the end of the string");
97
98 lt.pushBack();
99 assertEquals(12, lt.index, "Index should at the index of 'J'");
100 assertEquals("JKL", lt.nextToken(), "End of the string");
101 assertEquals(15, lt.index, "Index should be at the end of the string");
102 assertNull(lt.nextToken(), "No remaining tokens");
103 assertEquals(15, lt.index, "Index should not have changed");
104 }
105
106 }