1 package emissary.pickup.file;
2
3 import emissary.core.IBaseDataObject;
4 import emissary.core.Namespace;
5 import emissary.pickup.Priority;
6 import emissary.pickup.PriorityDirectory;
7 import emissary.pickup.WorkBundle;
8 import emissary.pickup.WorkSpace;
9 import emissary.test.core.junit5.FunctionalTest;
10 import emissary.util.io.ResourceReader;
11
12 import jakarta.annotation.Nullable;
13 import org.apache.commons.io.FileUtils;
14 import org.junit.jupiter.api.AfterEach;
15 import org.junit.jupiter.api.BeforeEach;
16 import org.junit.jupiter.api.Test;
17
18 import java.io.File;
19 import java.io.FileOutputStream;
20 import java.io.IOException;
21 import java.io.InputStream;
22
23 import static org.junit.jupiter.api.Assertions.assertEquals;
24 import static org.junit.jupiter.api.Assertions.assertNotNull;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
26 import static org.junit.jupiter.api.Assertions.fail;
27
28 class FTestSpacePlaceInteraction extends FunctionalTest {
29 @Nullable
30 private FilePickUpClient place = null;
31 @Nullable
32 private WorkSpace space = null;
33 @Nullable
34 private InputStream configStream = null;
35
36
37 private File inarea;
38 private File outarea;
39
40 @Override
41 @BeforeEach
42 public void setUp() throws Exception {
43 logger.debug("Starting WorkSpace tests");
44
45
46 inarea = new File(TMPDIR + "/filepicktest/in");
47 inarea.mkdirs();
48
49 outarea = new File(TMPDIR + "/filepicktest/out");
50 outarea.mkdirs();
51
52 File testfile = File.createTempFile("temp", ".dat", inarea);
53 testfile.deleteOnExit();
54
55 File inareadir = new File(inarea, "subdir");
56 inareadir.mkdirs();
57 inareadir.deleteOnExit();
58 File testfile2 = File.createTempFile("temp", ".dat", inareadir);
59 testfile2.deleteOnExit();
60 FileOutputStream os = new FileOutputStream(testfile);
61 os.write("This is a test".getBytes());
62 os.close();
63 os = new FileOutputStream(testfile2);
64 os.write("This is a test".getBytes());
65 os.close();
66
67
68 startJetty(8005);
69 logger.debug("WorkSpace test setup completed");
70 }
71
72 @Override
73 @AfterEach
74 public void tearDown() throws Exception {
75
76 logger.debug("Starting tearDown phase");
77
78
79
80
81
82
83
84 if (space != null) {
85 space.stop();
86 space = null;
87 }
88
89 if (place != null) {
90 place.shutDown();
91 place = null;
92 }
93
94 demolishServer();
95 restoreConfig();
96
97
98 FileUtils.deleteDirectory(inarea.getParentFile());
99 FileUtils.deleteDirectory(outarea);
100 FileUtils.deleteDirectory(new File(TMPDIR + "/data"));
101 }
102
103 private void createWorkspace(boolean useFileTimestamps, boolean useSimple) throws Exception {
104
105 space = new WorkSpace();
106 space.setEatPrefix(TMPDIR + "/filepicktest/in");
107 space.setOutputRoot(TMPDIR + "/filepicktest/out");
108 space.setCaseId("testcase");
109 space.setPattern("*.FILE_PICK_UP.INPUT.*");
110 space.setLoop(true);
111 space.setPauseTime(10);
112
113 space.setRetryStrategy(true);
114 space.addDirectory(new PriorityDirectory(inarea.getPath(), Priority.DEFAULT));
115 space.setDirectoryProcessing(false);
116 space.setFileTimestampUsage(useFileTimestamps);
117 space.setSimpleMode(useSimple);
118
119 assertTrue(Namespace.exists("http://localhost:8005/WorkSpace"), "WorkSpace should exist in namespace");
120
121 assertEquals(0, space.getFilesProcessed(), "No files proessed");
122 assertEquals(0, space.getBytesProcessed(), "No bytes proessed");
123 assertEquals(0, space.getBundlesProcessed(), "No bundles proessed");
124 assertEquals(1, space.getPickUpPlaceCount(), "Found pickup place");
125
126 }
127
128 @SuppressWarnings("ThreadPriorityCheck")
129 private void detachWorkspace(String threadName) {
130
131 Thread tspacethr = new Thread(space, threadName);
132 tspacethr.setDaemon(false);
133 tspacethr.start();
134 logger.debug("WorkSpace is started and detached on thread " + threadName);
135 Thread.yield();
136 }
137
138 @Test
139 void testUsingFileSeenList() throws Exception {
140
141 configStream = new ResourceReader().getConfigDataAsStream(this);
142 place = (FilePickUpClient) addPlace("http://localhost:8005/FilePickUpClient", "emissary.pickup.file.FilePickUpClient", configStream);
143 createWorkspace(false, false);
144 detachWorkspace("WorkSpaceTestSPI-testUsingFileSeenList");
145 runTest(800L, 1000L, 200L, 100L, false);
146 }
147
148 @Test
149 void testUsingFileTimeStamps() throws Exception {
150
151 configStream = new ResourceReader().getConfigDataAsStream(this);
152 place = (FilePickUpClient) addPlace("http://localhost:8005/FilePickUpClient", "emissary.pickup.file.FilePickUpClient", configStream);
153 createWorkspace(true, false);
154 detachWorkspace("WorkSpaceTestSPI-testUsingFileTimeStamps");
155 runTest(800L, 1000L, 200L, 100L, false);
156 }
157
158 @Test
159 void testUsingSimpleMode() throws Exception {
160
161 configStream = new ResourceReader().getConfigDataAsStream(this);
162 MyFilePickUpClient myplace =
163 (MyFilePickUpClient) addPlace("http://localhost:8005/MyFilePickUpClient",
164 "emissary.pickup.file.FTestSpacePlaceInteraction$MyFilePickUpClient", configStream);
165 place = myplace;
166 createWorkspace(true, true);
167 detachWorkspace("WorkSpaceTestSPI-testUsingSimpleMode");
168 runTest(800L, 1500L, 200L, 100L, true);
169 assertEquals(0,
170 myplace.expectationsNotMetCount,
171 "PickupPlace should have created payload objects that mirror WorkSpace simple setting with no mismatches");
172 }
173
174 private void runTest(long t1, long t2, long t3, long t4, boolean simple) {
175 pause(t1);
176
177 assertEquals(2, space.getFilesProcessed(), "File has been processed");
178 assertEquals(1, space.getBundlesProcessed(), "Bundle was created");
179 assertEquals("This is a test".length() * 2L, space.getBytesProcessed(), "Byte count matches file");
180
181 pause(t2);
182
183 assertEquals(0, space.getOutboundQueueSize(), "Outbound queue should empty");
184 assertEquals(0, space.getPendingQueueSize(), "Pending queue should empty");
185 logger.debug("Shutting down file pick up client");
186 place.shutDown();
187 place = null;
188 assertTrue(Namespace.exists("http://localhost:8005/WorkSpace"), "WorkSpace should exist in namespace");
189
190 pause(t3);
191 assertEquals(0, space.getPickUpPlaceCount(), "PickupPlace count adjusted");
192
193
194
195 String fakeKey = "THIS.IS.MY_KEY.http://localhost:8005/TestRunner";
196 try {
197 File extrafile = File.createTempFile("temp", ".dat", inarea);
198 extrafile.deleteOnExit();
199 logger.debug("Created new input file at " + extrafile.getPath());
200 FileOutputStream os = new FileOutputStream(extrafile);
201 String extramsg = "This is an extra test file";
202 os.write(extramsg.getBytes());
203 os.close();
204 pause(t4);
205
206
207 assertEquals(3, space.getFilesProcessed(), "Files proessed notices new file");
208
209
210
211 WorkBundle bundle = space.take(fakeKey);
212 pause(t4);
213
214 assertNotNull(bundle, "Bundle should be retreived with direct call");
215 assertEquals(1, space.getPendingQueueSize(), "File marked pending");
216
217 assertEquals(1, bundle.size(), "Bundle should contain the one extra file");
218 assertEquals(extrafile.getPath(), bundle.getFileNameList().get(0), "Bundle should match extra file name");
219 assertEquals(simple, bundle.getSimpleMode(), "Bundle should mirror work space simple mode");
220 space.workCompleted(fakeKey, bundle.getBundleId(), true);
221 pause(t4);
222
223 assertEquals(0, space.getPendingQueueSize(), "File no lnger marked pending");
224 } catch (IOException ex) {
225 fail("Cannot create extra test file", ex);
226 }
227
228
229 configStream = new ResourceReader().getConfigDataAsStream(this);
230 place = (FilePickUpClient) addPlace("http://localhost:8005/FilePickUpClient", "emissary.pickup.file.FilePickUpClient", configStream);
231
232 pause(t4);
233 assertEquals(1, space.getPickUpPlaceCount(), "Refound pickup place");
234
235
236 WorkBundle b;
237 do {
238 b = space.take(fakeKey);
239 if (b == null || b.size() == 0) {
240 break;
241 }
242 } while (b != null);
243
244 logger.debug("SpacePlace all tests completed!");
245
246 }
247
248 public static class MyFilePickUpClient extends FilePickUpClient {
249
250 boolean expectSimple = true;
251 public int expectationsMetCount = 0;
252 public int expectationsNotMetCount = 0;
253
254 public MyFilePickUpClient(InputStream configInfo, String dir, String placeLoc) throws IOException {
255 super(configInfo, dir, placeLoc);
256 }
257
258 public void setSimpleExpected(boolean value) {
259 expectSimple = value;
260 }
261
262 @Override
263 protected void dataObjectCreated(IBaseDataObject d, File f) {
264 super.dataObjectCreated(d, f);
265 boolean foundSimple = Boolean.parseBoolean(d.getStringParameter("SIMPLE_MODE"));
266 if (expectSimple == foundSimple) {
267 expectationsMetCount++;
268 } else {
269 expectationsNotMetCount++;
270 }
271 assertEquals(expectSimple, foundSimple, "Payloads should mirror forensic expectations");
272 }
273 }
274 }