kff = new KffDataObjectHandler(true, true, true);
        payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
        payload.setChannelFactory(SBC_DATA);
        kff.hash(payload);
        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
        assertEquals("test.filter", payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
        assertEquals("test.filter", payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
        assertTrue(KffDataObjectHandler.hashPresent(payload));
        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertEquals(DATA_CRC32, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());

kff = new KffDataObjectHandler(true, true, true);
        payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
        kff.hash(payload);
        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertTrue(KffDataObjectHandler.hashPresent(payload));
        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));

payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
        kff.hash(payload);
        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertTrue(KffDataObjectHandler.hashPresent(payload));
        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());

"MD5_ORIGINAL should only be populated if hashing more than once");

        // hash again, to see the effect on the hash-related params.
        // none of the parameters should have a duplicated value

        kff.hash(payload);
        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertEquals("test.filter", payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME));
        assertTrue(KffDataObjectHandler.hashPresent(payload));
        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_MD5, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));

, 10, -1, -1, -1, 13, -1, 116, -1, 118, -1 // 110
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 120
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 130
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 140
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 150
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 160
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 170
            , -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 180

place = new CoordinationPlace(configStream);
        config = ConfigUtil.getConfigInfo(new ResourceReader().getConfigDataAsStream(this));
    }

    @Override
    @AfterEach
    public void tearDown() throws Exception {
        demolishServer();
    }

    @Test
    void testRequiredPlaceCreation() throws Exception {
        for (String key : config.findEntries("SERVICE_COORDINATION")) {
            assertNotNull(Namespace.lookup(key)); // will throw exception if not present
        }

    }

    @Test
    void testProcessing() throws Exception {
        IBaseDataObject payload = DataObjectFactory.getInstance("test data".getBytes(), "test.dat", place.getPrimaryProxy());
        place.processHeavyDuty(payload);
        assertEquals(config.findStringEntry(OUTPUT_FORM), payload.currentForm(), "Current form must be set by coordinate place");
    }

    @Test
    void testProcessingWithResourceTracking() throws Exception {
        IBaseDataObject payload = DataObjectFactory.getInstance("test data".getBytes(), "test.dat", place.getPrimaryProxy());
        ResourceWatcher rw = new ResourceWatcher();
        place.processHeavyDuty(payload);
        assertEquals(config.findStringEntry(OUTPUT_FORM), payload.currentForm(), "Current form must be set by coordinate place");
        Map<String, Timer> resourcesUsed = rw.getStats();
        rw.quit();

private static final int[] trailingBytesForUtF8 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
            0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

assertEquals("test.filter", payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
        assertFalse(KffDataObjectHandler.hashPresent(payload));
        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
        assertNull(payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertNull(payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_MD5));
        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertNull(payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertNull(payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertNull(payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertNull(payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertNull(payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertNull(payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertNull(payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertNull(payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));

config.addEntry("EXTRA_PARAM", "*");
        f.initialize(config, "FOO", config);

        IBaseDataObject payload = DataObjectFactory.getInstance();
        payload.setData("This is the data".getBytes());
        payload.setFileType("FTYPE");
        payload.setFilename("/this/is/a/testfile");
        payload.putParameter("FOO", "myFoo");
        payload.putParameter("BAR", "myBar");
        payload.putParameter("BAR_AS_PREFIX", "myBarPrefix");
        payload.putParameter("QUUX", "myQuux");
        List<IBaseDataObject> payloadList = new ArrayList<>();
        payloadList.add(payload);

        Map<String, Object> params = new HashMap<>();

        assertTrue(f.isOutputtable(payload, params), "Payload should be outputtable");
        assertTrue(f.isOutputtable(payloadList, params), "Payload list should be outputtable");

        ByteArrayOutputStream output = new ByteArrayOutputStream();
        int status = f.filter(payloadList, params, output);
        String s = output.toString();

        assertEquals(IDropOffFilter.STATUS_SUCCESS, status, "Filter should return success");
        assertTrue(s.contains("\"QUUX\":[\"myQuux\"]"), "Filter output should have allowlist field");
        assertTrue(s.contains("\"BAR\":[\"myBar\"]"), "Filter output should have prefix no-match field BAR");

String[] args = {"--sort", "sf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(i);
            mws.addOutboundBundle(wb);

        }

        WorkBundle taken;
        int pri = 0;
        assertEquals(11, mws.getOutboundQueueSize(), "Outbound Queue Size");
        while ((taken = mws.take(C1)) != null) {
            if (taken.size() != 0) {
                assertEquals(pri, taken.getPriority(), "Ignoring size, priority work must be taken first");
            } else {
                break; // size = 0 means its the last
            }
            pri++;
        }
    }

    /**
     * Test the WorkBundle sorting of the "largest first" sorting when the priorities are even (and hence, don't override
     * the time based ordering)
     */
    @Test
    void testLargestFirstQueueing() throws Exception {

assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());
        assertArrayEquals(new byte[0], payload.data());
    }

    @Test
    void testHashMethodCalledTwice() {

AgentRule rule = new AllMaxTime("rule1", "TestPlace", 2, 0.75);
        assertFalse(rule.overTimeLimit(placeAgentStats));
    }

    @Nested
    class ConditionTest extends UnitTest {

        final String toUpperLowerPattern = "To(?:Lower|Upper)Place";
        final String toLowerPlace = "ToLowerPlace";
        final String toUpperPlace = "ToUpperPlace";
        final int defaultPoolSize = 5;
        final int defaultTimeLimit = 5;

        AgentPool pool;
        List<AgentProtocol.PlaceAgentStats> stats;

        @Override
        @BeforeEach
        public void setUp() throws Exception {
            super.setUp();
            pool = mock(AgentPool.class);
            stats = stats();
        }

        @Test
        void condition1() {
            assertTrue(testRule(toUpperLowerPattern, defaultTimeLimit, 1.0, defaultPoolSize));
        }

        @Test
        void condition2() {
            assertFalse(testRule(toUpperLowerPattern, defaultTimeLimit, 1.0, defaultPoolSize + 1));
        }

        @Test
        void condition3() {

assertEquals(W, new String(payload.data()).trim(), "Clean UTF-8 coming from script must be in parent");
        assertEquals(1, payload.currentFormSize(), "Single form remaining for parent");
        assertEquals(1, att.get(0).currentFormSize(), "Single form for child");

        assertEquals("copy value", att.get(0).getStringParameter("COPY_THIS"), "Child should have propagating metadata value");
        assertEquals("copy value", att.get(0).getParameterAsConcatString("COPY_THIS"), "Child should have propagating metadata value");
        assertEquals("copy value", att.get(0).getParameterAsString("COPY_THIS"), "Child should have propagating metadata value");
        assertNull(att.get(0).getStringParameter("IGNORE_THIS"), "Child should not have non-propagating metadata value");
        assertNull(att.get(0).getParameterAsConcatString("IGNORE_THIS"), "Child should not have non-propagating metadata value");
        assertNull(att.get(0).getParameterAsString("IGNORE_THIS"), "Child should not have non-propagating metadata value");
    }

    @Test
    void testMultiFileUnixCommandPlaceFile() throws Exception {

assertEquals(DATA_CRC32, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_CRC32));
        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SSDEEP, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SSDEEP));
        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA1, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA1));
        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsConcatString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(DATA_SHA256, payload.getParameterAsString(KffDataObjectHandler.KFF_PARAM_SHA256));
        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());

assertNotNull(this.util.getBestId(this.payload, tld));
        assertEquals("yes", this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");

        // Test get ID from parameter //////////////////////////////
        cfg = new ServiceConfigGuide();

        ids = new ArrayList<>();
        ids.add("MY_ID");
        ids.add("SHORTNAME");
        ids.add("AUTO_GENERATED_ID");
        cfg.addEntries("ID", ids);

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "/eat/prefix/testPath", "UNKNOWN");
        this.payload.setParameter("MY_ID", "672317892139");
        this.util = new DropOffUtil(cfg);
        id = this.util.getBestId(this.payload, tld);

id = this.util.getBestId(this.payload, tld);

        assertEquals("yes", this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertNotEquals("testPath", id, "the SHORTNAME parameter should NOT have been used");
        assertTrue(id.startsWith("ABCD"), "auto gen id should start with a (truncated to 4 char) prefix");

        // Test force auto gen by specifying nothing
        // //////////////////////////////

        cfg = new ServiceConfigGuide();

        ids = new ArrayList<>();
        cfg.addEntries("ID", ids);

        cfg.addEntry("AUTO_GENERATED_ID_PREFIX", "ABCDEFGH");

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "", "UNKNOWN");// shortname is blank

        this.util = new DropOffUtil(cfg);
        id = this.util.getBestId(this.payload, tld);

final int ch = this.data[pos] & 0x7f;
            final int jump = this.lookup[ch];
            BackwardsTreeScanner.State state = this.root.nextStates[ch];
            int curPos = pos - 1;
            while ((state != null) && (curPos >= 0)) {
                if (state.matches != null) {
                    for (int i = 0; i < state.matches.length; i++) {
                        final int id = state.matches[i];
                        final int[] tmp = new int[3];
                        tmp[0] = curPos + 1;
                        tmp[1] = id;
                        tmp[2] = pos - curPos;
                        result.add(tmp);

setAllFieldsPrintable(ibdo, bytes);

        final String xmlString = IBaseDataObjectXmlHelper.xmlFromIbdo(ibdo, children, ibdo, DEFAULT_ELEMENT_ENCODERS);

        final SAXBuilder builder = new SAXBuilder(XMLReaders.NONVALIDATING);
        final Document document = builder.build(new StringReader(xmlString));
        final Element root = document.getRootElement();

        assertNull(checkLengthElement(root, (int) ibdo.getChannelSize(), "answers", "data"));
        assertNull(checkLengthElement(root, ibdo.footer().length, "answers", "footer"));
        assertNull(checkLengthElement(root, ibdo.header().length, "answers", "header"));
        assertNull(checkLengthKeyValue(root, ibdo.getAlternateView("AlternateView1Key").length, "AlternateView1Key", "answers", "view"));

chain.addAlgorithm("SHA-256");

        assertEquals(3, chain.getAlgorithms().size(), "Algorithms stored in chain");

        try {
            ChecksumResults cr = chain.computeSums(DATA);
            Set<String> algs = cr.getResultsPresent();
            assertNotNull(algs, "Algorithm set returned");
            assertEquals(3, algs.size(), "All results present");
            Iterator<String> i = algs.iterator();
            assertEquals("MD5", i.next(), "MD5 alg present");
            assertEquals("SHA-1", i.next(), "SHA-1 alg present");
            assertEquals("SHA-256", i.next(), "SHA-256 alg present");
        } catch (Exception ex) {
            fail("Could not compute results", ex);
        }
    }

    @Test
    void testAlgorithmsUsedWithSetAlgorithm() {

ids.add("SHORTNAME");
        cfg.addEntries("ID", ids);

        cfg.addEntry("AUTO_GENERATED_ID_PREFIX", "ABCDEFGH");

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "", "UNKNOWN");// shortname is blank

        this.util = new DropOffUtil(cfg);
        id = this.util.getBestId(this.payload, tld);

        assertEquals("yes", this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertNotEquals("testPath", id, "the SHORTNAME parameter should NOT have been used");
        assertTrue(id.startsWith("ABCD"), "auto gen id should start with a (truncated to 4 char) prefix");

ids.add("SHORTNAME");
        cfg.addEntries("ID", ids);

        cfg.addEntry("AUTO_GENERATED_ID_PREFIX", "ABCDEFGH");

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "", "UNKNOWN");// shortname is blank

        this.util = new DropOffUtil(cfg);
        id = this.util.getBestIdFrom(this.payload);

        assertEquals("yes", this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertEquals("yes", this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should have been set");
        assertNotEquals("testPath", id, "the SHORTNAME parameter should NOT have been used");
        assertTrue(id.startsWith("ABCD"), "auto gen id should start with a (truncated to 4 char) prefix");

this.d.proxyFor(proxyKey);
        assertEquals(cost, this.d.getCost(), "Proxy cost");
        assertEquals(quality, this.d.getQuality(), "Proxy qual");
        assertEquals(KeyManipulator.getServiceType(key), this.d.getServiceType(), "Proxy service type");
        assertEquals(KeyManipulator.getServiceName(key), this.d.getServiceName(), "Proxy service name");
        assertEquals(KeyManipulator.getDataType(key), KeyManipulator.getDataType(this.d.getKey()), "Proxy data type");
        assertEquals(DirectoryEntry.calculateExpense(cost, quality), this.d.getExpense(), "Proxy expense");

void testOldestFirstQueueing() throws Exception {
        String[] args = {"--sort", "of", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] times = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < times.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", times[i], 5L);
            wb.setPriority(1); // uniform priority to ensure times are used, not priority

void testYoungestFirstQueueing() throws Exception {
        String[] args = {"--sort", "yf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] times = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < times.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", times[i], 2L);
            wb.setPriority(1); // uniform priority to make sure time sorting works

void testSmallestFirstQueueing() throws Exception {
        String[] args = {"--sort", "sf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(1); // uniform priority, so the time ordering is used

String[] args = {"--sort", "sf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(1); // uniform priority, so the time ordering is used
            mws.addOutboundBundle(wb);

        }

        // This should be in order of 0,1,2,3.....10
        WorkBundle taken;
        long sz = 0L;

void testLargestFirstQueueing() throws Exception {
        String[] args = {"--sort", "lf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(1); // uniform priority to sort by size

id = this.util.getBestId(this.payload, tld);

        assertNull(this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertNull(this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertNull(this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertEquals("testPath", id, "the SHORTNAME parameter should have been used");

        // Test force auto gen by specifying a blank shortname
        // //////////////////////////////

        cfg = new ServiceConfigGuide();

        ids = new ArrayList<>();
        ids.add("SHORTNAME");
        cfg.addEntries("ID", ids);

        cfg.addEntry("AUTO_GENERATED_ID_PREFIX", "ABCDEFGH");

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "", "UNKNOWN");// shortname is blank

        this.util = new DropOffUtil(cfg);
        id = this.util.getBestId(this.payload, tld);

Configurator cfg = new ServiceConfigGuide();
        final List<String> dates = new ArrayList<>();
        dates.add(EVENT_DATE);
        dates.add(FILE_DATE);
        cfg.addEntries("DATE_FORMAT", dates);

        List<String> ids = new ArrayList<>();
        ids.add("AUTO_GENERATED_ID");
        cfg.addEntries("ID", ids);

        cfg.addEntry("DATE_FORMAT", "E, d MMM yyyy HH:mm:ss Z");

        cfg.addEntry("AUTO_GENERATED_ID_PREFIX", "ABCDEFGH");

        this.util = new DropOffUtil(cfg);
        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "/eat/prefix/testPath", "UNKNOWN");
        String id = this.util.getBestId(this.payload, tld);

id = this.util.getBestId(this.payload, tld);

        assertNull(this.payload.getStringParameter("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertNull(this.payload.getParameterAsConcatString("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertNull(this.payload.getParameterAsString("AUTO_GENERATED_ID"), "an auto gen id parameter should NOT have been set");
        assertEquals("672317892139", id, "the MY_ID parameter should have been used");

        // Test shortname //////////////////////////////
        cfg = new ServiceConfigGuide();

        ids = new ArrayList<>();
        ids.add("SHORTNAME");
        ids.add("AUTO_GENERATED_ID");
        cfg.addEntries("ID", ids);

        this.payload = DataObjectFactory.getInstance("This is a test".getBytes(), "/eat/prefix/testPath", "UNKNOWN");

        this.util = new DropOffUtil(cfg);
        id = this.util.getBestId(this.payload, tld);

if (data[i] == '\\' && (i + 5) < data.length && data[i + 1] == 'u') {
                // process unicode escape
                try {
                    String s = new String(data, i + 2, 4);
                    char[] c = HtmlEscape.unescapeHtmlChar(s, true);
                    if (c != null && c.length > 0) {
                        out.write(new String(c).getBytes());
                        // logger.debug("Unicode '{}' ==> '{}'", s, new String(c));
                        i += 5;
                    } else {
                        out.write(data[i]);
                    }
                } catch (IOException iox) {
                    out.write(data[i]);
                }
            } else if (data[i] == '\\' && (i + 1) < data.length && (data[i + 1] == 'n' || data[i + 1] == 'r')) {

w.addFileName("<file2.txt&foo=bar>", 7L, 10L);
        w.setPriority(1);
        w.setSimpleMode(false);

        String xml = null;
        try {
            xml = w.toXml();
        } catch (RuntimeException ex) {
            fail("Cannot generate xml", ex);
        }
        assertNotNull(xml, "Generated xml");
        WorkBundle w2 = WorkBundle.buildWorkBundle(xml);
        assertNotNull(w2, "Generated from xml");
        assertEquals(w.getOutputRoot(), w2.getOutputRoot(), "Value across xml");
        assertEquals(w.getEatPrefix(), w2.getEatPrefix(), "Value across xml");
        assertEquals(w.getCaseId(), w2.getCaseId(), "Value across xml");
        assertEquals(w.getYoungestFileModificationTime(), w2.getYoungestFileModificationTime(), "Youngest time across xml");

try (Response response = target(DEREGISTER_PLACE_ACTION).request().post(Entity.form(formParams))) {
            // verify
            final int status = response.getStatus();
            assertEquals(500, status);
            final String result = response.readEntity(String.class);
            assertTrue(result.startsWith("Bad params:"));
        }
    }

    @ParameterizedTest
    @ValueSource(strings = {"test\n\r", "\n", "\r"})
    void cleanParams(String paramsToSanitize) {
        // setup
        formParams.replace(TARGET_DIRECTORY, Collections.singletonList(paramsToSanitize));
        formParams.replace(ADD_KEY, Collections.singletonList(paramsToSanitize));

        // test
        try (Response response = target(DEREGISTER_PLACE_ACTION).request().post(Entity.form(formParams))) {

private static final byte[] expectedSha1Bytes = {(byte) 0, (byte) 0, (byte) 0, (byte) 32, (byte) 103, (byte) 56, (byte) 116,
            (byte) -114, (byte) -35, (byte) -110, (byte) -60, (byte) -29, (byte) -46, (byte) -24, (byte) 35, (byte) -119,
            (byte) 103, (byte) 0, (byte) -8, (byte) 73};

outarea1 = new File(TMPDIR + "/multipicktest/space1/out");
        outarea1.mkdirs();

        holdarea = new File(TMPDIR, "/data/HoldData");
        holdarea.mkdirs();

        File testfile = File.createTempFile("temp1", ".dat", inarea1);
        workingFiles.add(testfile);
        workingFilePaths.add(testfile.getName());
        testfile.deleteOnExit();

        inareadir1 = new File(inarea1, "subdir1");
        inareadir1.mkdirs();
        inareadir1.deleteOnExit();
        File testfile2 = File.createTempFile("temp2", ".dat", inareadir1);
        workingFiles.add(testfile2);
        workingFilePaths.add("subdir1/" + testfile2.getName());
        testfile2.deleteOnExit();

w.addFileName("<file2.txt&foo=bar>", 7L, 10L);
        w.setPriority(1);
        w.setSimpleMode(false);

        String xml = null;
        try {
            xml = w.toXml();
        } catch (RuntimeException ex) {
            fail("Cannot generate xml", ex);
        }
        assertNotNull(xml, "Generated xml");
        WorkBundle w2 = WorkBundle.buildWorkBundle(xml);
        assertNotNull(w2, "Generated from xml");
        assertEquals(w.getOutputRoot(), w2.getOutputRoot(), "Value across xml");
        assertEquals(w.getEatPrefix(), w2.getEatPrefix(), "Value across xml");
        assertEquals(w.getCaseId(), w2.getCaseId(), "Value across xml");

assertEquals(w.getTotalFileSize(), w2.getTotalFileSize(), "Total filesize across xml");
        assertEquals(w.size(), w2.size(), "Files across xml");
        assertEquals(w.getPriority(), w2.getPriority(), "Priority across xml");
        assertEquals(w.getSimpleMode(), w2.getSimpleMode(), "Simple mode across xml");
        List<String> w2l = w2.getFileNameList();
        assertNotNull(w2l, "File list from xml");
        assertEquals(2, w2l.size(), "Size of file list from xml");
        assertEquals("file1.txt", w2l.get(0), "File values from xml");
        assertEquals("<file2.txt&foo=bar>", w2l.get(1), "File values from xml");
    }

    @Test
    void testBundleXmlWithDefaultTimes() {

String[] args = {"--sort", "sf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(1); // uniform priority, so the time ordering is used

String[] args = {"--sort", "sf", "-i", "blah"};
        mws = new MyWorkSpace(FeedCommand.parse(FeedCommand.class, args));

        String C1 = "INITIAL.INPUT.A.http://otherhost:7001/FilePickUpClient";
        mws.addPickUp(C1);

        long[] sizes = {4, 5, 3, 1, 2, 7, 0, 8, 9, 10, 6};
        for (int i = 0; i < sizes.length; i++) {
            // Set up some fake work
            WorkBundle wb = new WorkBundle("/fake/root", "/fake/eat");
            wb.addFileName("faker-" + i + ".txt", 2L, sizes[i]);
            wb.setPriority(i);

payload.appendParameter("BAR", "foo");

        List<IBaseDataObject> payloadList = new ArrayList<>();
        payloadList.add(payload);

        Map<String, Object> params = new HashMap<>();

        assertTrue(f.isOutputtable(payload, params), "Payload should be outputtable");
        assertTrue(f.isOutputtable(payloadList, params), "Payload list should be outputtable");

        ByteArrayOutputStream output = new ByteArrayOutputStream();
        int status = f.filter(payloadList, params, output);
        String s = output.toString();

        assertEquals(IDropOffFilter.STATUS_SUCCESS, status, "Filter should return success");
        assertTrue(s.contains("\"BAR\":[\"foo\"]"), "Filter output should contain field BAR");

private static void check(final ListAppender<ILoggingEvent> appender, final String message,
            final boolean testStackTrace) {
        assertEquals(1, appender.list.size());

        final ILoggingEvent loggingEvent = appender.list.get(0);

        assertEquals(message, loggingEvent.getFormattedMessage());

        final IThrowableProxy throwableProxy = loggingEvent.getThrowableProxy();

        if (testStackTrace) {
            assertNotNull(throwableProxy);
            assertEquals("java.lang.Throwable", throwableProxy.getClassName());
            assertEquals("DEBUG", throwableProxy.getMessage());
        } else {
            assertNull(throwableProxy);
        }

        appender.list.clear();
    }

    private static class TestFileChannelFactory implements SeekableByteChannelFactory {