View Javadoc
1   package emissary.pickup;
2   
3   import emissary.command.FeedCommand;
4   import emissary.core.Namespace;
5   import emissary.directory.EmissaryNode;
6   import emissary.directory.IDirectoryPlace;
7   import emissary.pickup.file.FilePickUpClient;
8   import emissary.test.core.junit5.FunctionalTest;
9   import emissary.util.shell.Executrix;
10  
11  import org.junit.jupiter.api.AfterEach;
12  import org.junit.jupiter.api.BeforeEach;
13  import org.junit.jupiter.api.Test;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.util.ArrayList;
18  import java.util.List;
19  import javax.annotation.Nullable;
20  
21  import static org.junit.jupiter.api.Assertions.assertEquals;
22  import static org.junit.jupiter.api.Assertions.assertFalse;
23  import static org.junit.jupiter.api.Assertions.assertTrue;
24  
25  class FTestMultipleWorkSpaces extends FunctionalTest {
26      @Nullable
27      private FilePickUpClient place = null;
28      @Nullable
29      private WorkSpace space1 = null;
30      @Nullable
31      private WorkSpace space2 = null;
32      @Nullable
33      private IDirectoryPlace peer2 = null;
34  
35      // Workspace input and output directories
36      private File inarea1;
37      private File inareadir1;
38      private File outarea1;
39  
40      private File inarea2;
41      private File inareadir2;
42      private File outarea2;
43  
44      private File holdarea;
45  
46      private final List<File> workingFiles = new ArrayList<>();
47      private final List<String> workingFilePaths = new ArrayList<>();
48  
49      @Override
50      @BeforeEach
51      public void setUp() throws Exception {
52          logger.debug("Starting WorkSpace tests");
53  
54          // Set up a directory struction with two files to be processed
55          // rom each workspace
56          inarea1 = new File(TMPDIR + "/multipicktest/space1/in");
57          inarea1.mkdirs();
58  
59          outarea1 = new File(TMPDIR + "/multipicktest/space1/out");
60          outarea1.mkdirs();
61  
62          holdarea = new File(TMPDIR, "/data/HoldData");
63          holdarea.mkdirs();
64  
65          File testfile = File.createTempFile("temp1", ".dat", inarea1);
66          workingFiles.add(testfile);
67          workingFilePaths.add(testfile.getName());
68          testfile.deleteOnExit();
69  
70          inareadir1 = new File(inarea1, "subdir1");
71          inareadir1.mkdirs();
72          inareadir1.deleteOnExit();
73          File testfile2 = File.createTempFile("temp2", ".dat", inareadir1);
74          workingFiles.add(testfile2);
75          workingFilePaths.add("subdir1/" + testfile2.getName());
76          testfile2.deleteOnExit();
77          FileOutputStream os = new FileOutputStream(testfile);
78          os.write("This is a test".getBytes());
79          os.close();
80          os = new FileOutputStream(testfile2);
81          os.write("This is a test".getBytes());
82          os.close();
83  
84          inarea2 = new File(TMPDIR + "/multipicktest/space2/in");
85          inarea2.mkdirs();
86  
87          outarea2 = new File(TMPDIR + "/multipicktest/space2/out");
88          outarea2.mkdirs();
89  
90          File testfile3 = File.createTempFile("temp3", ".dat", inarea2);
91          workingFiles.add(testfile3);
92          workingFilePaths.add(testfile3.getName());
93          testfile3.deleteOnExit();
94  
95          inareadir2 = new File(inarea2, "subdir2");
96          inareadir2.mkdirs();
97          inareadir2.deleteOnExit();
98          File testfile4 = File.createTempFile("temp2", ".dat", inareadir2);
99          workingFiles.add(testfile4);
100         workingFilePaths.add("subdir2/" + testfile4.getName());
101         testfile4.deleteOnExit();
102         os = new FileOutputStream(testfile3);
103         os.write("This is a test".getBytes());
104         os.close();
105         os = new FileOutputStream(testfile4);
106         os.write("This is a test".getBytes());
107         os.close();
108 
109 
110         // start jetty and directory services
111         // TODO These FTestWorkSpace* tests will compile now but need to be totally reworked due to the way we
112         // start/stop emissary
113         startJetty(8005);
114         // Start a second client to keep things happy
115         peer2 = startDirectory(9005);
116         peer2.heartbeatRemoteDirectory(directory.getKey());
117         directory.heartbeatRemoteDirectory(peer2.getKey());
118 
119         // Start a FilePickUpClient
120         place = (FilePickUpClient) addPlace("http://localhost:8005/FilePickUpClient", FilePickUpClient.class.getName());
121 
122         System.setProperty(EmissaryNode.NODE_PORT_PROPERTY, "" + 8005);
123 
124         // Create and configure a WorkSpace
125         space1 =
126                 new WorkSpace(FeedCommand.parse(FeedCommand.class, new String[] {"-ns", "WorkSpace1", "-c", TMPDIR, "-i",
127                         TMPDIR + "/multipicktest/space1/in:10"}));
128         space1.setEatPrefix(TMPDIR + "/multipicktest/space1/in");
129         space1.setOutputRoot(TMPDIR + "/multipicktest/space1/out");
130         space1.setCaseId("space1case");
131         space1.setLoop(false);
132         space1.setPauseTime(10);// millis
133         space1.setRetryStrategy(true);
134         space1.setDirectoryProcessing(false);
135 
136         // Create and configure a second WorkSpace
137         space2 =
138                 new WorkSpace(FeedCommand.parse(FeedCommand.class, new String[] {"-ns", "WorkSpace2", "-c", TMPDIR, "-i",
139                         TMPDIR + "/multipicktest/space2/in:10"}));
140         space2.setEatPrefix(TMPDIR + "/multipicktest/space2/in");
141         space2.setOutputRoot(TMPDIR + "/multipicktest/space2/out");
142         space2.setCaseId("space2case");
143         space2.setLoop(false);
144         space2.setPauseTime(10);// millis
145         space2.setRetryStrategy(true);
146         space2.setDirectoryProcessing(false);
147 
148         logger.debug("WorkSpace test setup completed");
149     }
150 
151     @Override
152     @AfterEach
153     public void tearDown() throws Exception {
154 
155         logger.debug("Starting tearDown phase");
156 
157         if (space1 != null) {
158             logger.debug("Space1 stats >> " + space1.getStatsMessage());
159             space1.stop();
160             space1 = null;
161         }
162 
163         if (space2 != null) {
164             logger.debug("Space2 stats >> " + space2.getStatsMessage());
165             space2.stop();
166             space2 = null;
167         }
168 
169         if (place != null) {
170             place.shutDown();
171             place = null;
172         }
173 
174         if (peer2 != null) {
175             peer2.shutDown();
176             peer2 = null;
177         }
178 
179         demolishServer();
180 
181         // Clean up directories
182         inareadir1.delete();
183         inarea1.delete();
184         outarea1.delete();
185         inarea1.getParentFile().delete();
186         inareadir2.delete();
187         inarea2.delete();
188         outarea2.delete();
189         inarea2.getParentFile().delete();
190 
191         super.tearDown();
192     }
193 
194     @Test
195     @SuppressWarnings("ThreadPriorityCheck")
196     void testAll() {
197 
198         assertTrue(Namespace.exists("http://localhost:8005/WorkSpace1"), "First WorkSpace should exist in namespace");
199 
200         assertTrue(Namespace.exists("http://localhost:8005/WorkSpace2"), "Second WorkSpace should exist in namespace");
201 
202         pause(100);
203 
204         int byteSize = 0;
205         int fileCount = 0;
206         int bundleCount = 0;
207         int clientCount = 1;
208         int expectedOutbound = 0;
209         int expectedPending = 0;
210         int expectedRetries = 0;
211 
212         checkFileCounts(space1, fileCount, byteSize, bundleCount, clientCount, expectedOutbound, expectedPending, expectedRetries);
213         checkFileCounts(space2, fileCount, byteSize, bundleCount, clientCount, expectedOutbound, expectedPending, expectedRetries);
214 
215         Thread tspacethr1 = new Thread(space1, "WorkSpace1Test");
216         tspacethr1.setDaemon(true);
217         tspacethr1.start();
218         Thread.yield();
219         Thread tspacethr2 = new Thread(space2, "WorkSpace2Test");
220         tspacethr2.setDaemon(true);
221         tspacethr2.start();
222         Thread.yield();
223 
224         logger.debug("WorkSpaces are both started!");
225 
226         pause(10000);
227 
228         // Only count half of the stuff
229         byteSize = "This is a test".length() * 2;
230         bundleCount++;
231         fileCount += 2;
232 
233         checkFileCounts(space1, fileCount, byteSize, bundleCount, clientCount, expectedOutbound, expectedPending, expectedRetries);
234         checkFileCounts(space2, fileCount, byteSize, bundleCount, clientCount, expectedOutbound, expectedPending, expectedRetries);
235 
236         pause(500);
237         checkFileLocations();
238         logger.debug("MultipleWorkSpace all tests completed!");
239     }
240 
241     private void checkFileLocations() {
242         // Detailed debugging help on the structure of what is left in the file system
243         if (logger.isDebugEnabled()) {
244             StringBuilder sb = new StringBuilder();
245             new Executrix().execute(new String[] {"find", TMPDIR + "/multipicktest", TMPDIR + "data", "-print"}, sb);
246             logger.debug("Files:\n" + sb);
247         }
248 
249         // Assert things about where the files are located
250         int counter = 0;
251         for (File f : workingFiles) {
252             assertFalse(f.exists(), "File[" + counter + "] should not exist in input area any more - " + f);
253             counter++;
254         }
255 
256         counter = 0;
257         for (String fn : workingFilePaths) {
258             File fp = new File(place.getInProcessArea() + "/" + fn);
259             File fd1 = new File(space1.getOutputRoot() + "/" + fn);
260             File fd2 = new File(space2.getOutputRoot() + "/" + fn);
261             assertFalse(fp.exists(), "File[" + counter + "] should not exist in in-process area any more - " + fn + " - " + fp.getPath());
262             assertTrue(
263                     fd1.exists() || fd2.exists(),
264                     "File[" + counter + "] should exist in one of the two output root areas - " + fn + " - " + fd1.getPath() + ", " + fd2.getPath());
265             counter++;
266         }
267     }
268 
269     private static void checkFileCounts(WorkSpace space, int files, int bytes, int bundles, int places, int outbound, int pending, int retried) {
270         assertEquals(files, space.getFilesProcessed(), "files processed on " + space.getKey());
271         assertEquals(bytes, space.getBytesProcessed(), "bytes processed on " + space.getKey());
272         assertEquals(bundles, space.getBundlesProcessed(), "bundles processed on " + space.getKey());
273         assertEquals(places, space.getPickUpPlaceCount(), "pickup place count in " + space.getKey());
274         assertEquals(outbound, space.getOutboundQueueSize(), "Outbound queue count in " + space.getKey());
275         assertEquals(pending, space.getPendingQueueSize(), "Pending queue count in " + space.getKey());
276         assertEquals(retried, space.getRetriedCount(), "Retried bundle count in " + space.getKey());
277     }
278 }