View Javadoc
1   package emissary.output;
2   
3   import emissary.config.ConfigUtil;
4   import emissary.config.Configurator;
5   import emissary.core.Family;
6   import emissary.core.IBaseDataObject;
7   import emissary.util.FlexibleDateTimeParser;
8   import emissary.util.ShortNameComparator;
9   import emissary.util.TimeUtil;
10  import emissary.util.shell.Executrix;
11  
12  import org.apache.commons.io.FilenameUtils;
13  import org.apache.commons.lang3.StringUtils;
14  import org.slf4j.Logger;
15  import org.slf4j.LoggerFactory;
16  
17  import java.io.File;
18  import java.io.IOException;
19  import java.nio.file.FileSystems;
20  import java.nio.file.Files;
21  import java.nio.file.Path;
22  import java.nio.file.Paths;
23  import java.security.SecureRandom;
24  import java.time.Instant;
25  import java.time.ZonedDateTime;
26  import java.time.format.DateTimeParseException;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collection;
30  import java.util.Date;
31  import java.util.HashMap;
32  import java.util.HashSet;
33  import java.util.List;
34  import java.util.Locale;
35  import java.util.Map;
36  import java.util.Set;
37  import java.util.UUID;
38  import javax.annotation.Nullable;
39  
40  import static emissary.core.Form.PREFIXES_LANG;
41  import static emissary.core.Form.TEXT;
42  import static emissary.core.Form.UNKNOWN;
43  import static emissary.core.constants.Parameters.FILEXT;
44  import static emissary.core.constants.Parameters.FILE_ABSOLUTEPATH;
45  import static emissary.core.constants.Parameters.ORIGINAL_FILENAME;
46  import static emissary.util.TimeUtil.DATE_ISO_8601;
47  import static emissary.util.TimeUtil.getDateOrdinalWithTime;
48  
49  public class DropOffUtil {
50      protected static final Logger logger = LoggerFactory.getLogger(DropOffUtil.class);
51  
52      protected static final String SEPARATOR = FileSystems.getDefault().getSeparator();
53      protected static final String OS_NAME = System.getProperty("os.name").toUpperCase(Locale.getDefault());
54  
55      protected String unixRoot;
56  
57      protected String placeOutputData;
58  
59      /** Sources for building an ID for an item */
60      protected List<String> idTokens = new ArrayList<>();
61  
62      /** Sources for building a date string for an item */
63      protected List<String> dateTokens = new ArrayList<>();
64  
65      /** params to get from parent and save as PARENT_param */
66      protected List<String> parentParams = new ArrayList<>();
67  
68      /** Output directories */
69      protected Map<String, File> outputDirectories = new HashMap<>();
70  
71      protected Executrix executrix;
72  
73      // Items for generating random filenames
74      protected static final SecureRandom prng = new SecureRandom();
75      protected static final byte[] ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes();
76      protected String prefix = "TXT";
77      protected boolean uuidInOutputFilenames = true;
78      protected int maxFilextLen = Integer.MAX_VALUE;
79  
80      // Items for generating UUIDs
81      private static final String AUTO_GENERATED_ID = "AUTO_GENERATED_ID";
82      private static final String PARENT_AUTO_GENERATED_ID = "PARENT_AUTO_GENERATED_ID";
83      @Nullable
84      private String autoGeneratedIdPrefix = null;
85  
86      protected static final String EXTENDED_FILETYPE = "EXTENDED_FILETYPE";
87      protected static final String PARENT_FILETYPE = "PARENT_FILETYPE";
88      protected static final String SHORTNAME = "SHORTNAME";
89      protected static final String TARGETBIN = "TARGETBIN";
90  
91      private static final String DEFAULT_EVENT_DATE_TO_NOW = "DEFAULT_EVENT_DATE_TO_NOW";
92      protected boolean defaultEventDateToNow = true;
93  
94      /**
95       * Create with the default configuration
96       */
97      public DropOffUtil() {
98          configure(null);
99      }
100 
101     /**
102      * Create with the specified configuration
103      */
104     public DropOffUtil(final Configurator configG) {
105         configure(configG);
106     }
107 
108     /**
109      * Set up config for this class
110      * <ul>
111      * <li>ID_PARAMETER : multiple parameter values, ordered list of how to build a EMISSARY_ID</li>
112      * <li>ID : backwards compatibility for ID_PARAMETER only used if ID_PARAMETER does not exist</li>
113      * <li>DATE_PARAMETER : multiple parameter values, ordered list of how to build a date path</li>
114      * <li>OUTPUT_FILE_PREFIX: string to use when generating random filenames, default: TXT</li>
115      * <li>UUID_IN_OUTPUT_FILENAMES: boolean [true]</li>
116      * <li>AUTO_GENERATED_ID_PREFIX: prefix to use for an auto-generated id</li>
117      * </ul>
118      */
119     protected void configure(@Nullable final Configurator configG) {
120         Configurator actualConfigG = configG;
121         if (actualConfigG == null) {
122             try {
123                 actualConfigG = ConfigUtil.getConfigInfo(DropOffUtil.class);
124             } catch (IOException e) {
125                 logger.error("Cannot open default config file", e);
126             }
127         }
128 
129         if (actualConfigG != null) {
130             this.placeOutputData = actualConfigG.findStringEntry("OUTPUT_DATA", "outputData");
131             this.unixRoot = actualConfigG.findStringEntry("UNIX_ROOT", null);
132             this.executrix = new Executrix(actualConfigG);
133             this.idTokens = actualConfigG.findEntries("ID_PARAMETER");
134             this.autoGeneratedIdPrefix = actualConfigG.findStringEntry("AUTO_GENERATED_ID_PREFIX");
135 
136             // truncate the prefix if necessary
137             if (!StringUtils.isBlank(this.autoGeneratedIdPrefix) && (this.autoGeneratedIdPrefix.length() > 4)) {
138                 this.autoGeneratedIdPrefix = this.autoGeneratedIdPrefix.substring(0, 4);
139             }
140             if (this.idTokens.isEmpty()) {
141                 this.idTokens = actualConfigG.findEntries("ID");
142             }
143             this.dateTokens = actualConfigG.findEntries("DATE_PARAMETER");
144             this.parentParams = actualConfigG.findEntries("PARENT_PARAM");
145 
146 
147             this.defaultEventDateToNow = actualConfigG.findBooleanEntry(DEFAULT_EVENT_DATE_TO_NOW, this.defaultEventDateToNow);
148 
149             this.prefix = actualConfigG.findStringEntry("OUTPUT_FILE_PREFIX", prefix);
150             this.uuidInOutputFilenames = actualConfigG.findBooleanEntry("UUID_IN_OUTPUT_FILENAMES", this.uuidInOutputFilenames);
151             // limit the length of the FILEXT param
152             this.maxFilextLen = actualConfigG.findIntEntry("MAX_FILEXT_LEN", this.maxFilextLen);
153             if (this.maxFilextLen < 0) {
154                 this.maxFilextLen = Integer.MAX_VALUE;
155             }
156 
157         } else {
158             logger.debug("Configuration is null for DropOffUtil, using defaults");
159             this.executrix = new Executrix();
160         }
161     }
162 
163     /**
164      * Generate a new random build file name using the configured prefix and strategy
165      */
166     public String generateBuildFileName() {
167         if (this.uuidInOutputFilenames) {
168             return (prefix + getDateOrdinalWithTime(Instant.now()) + UUID.randomUUID());
169         } else {
170             // Using some constants plus yyyyJJJhhmmss plus random digit, letter, digit
171             return (prefix + getDateOrdinalWithTime(Instant.now()) + prng.nextInt(10) + ALPHABET[prng.nextInt(ALPHABET.length)] + prng.nextInt(10));
172         }
173     }
174 
175     /**
176      * Drop off often needs to make way for a file it wants to write Do so and return the name of the file that can now be
177      * written
178      */
179     public String makeWayForIncomingFile(final IBaseDataObject ibdo, final String suffix) {
180         return makeWayForIncomingFile(ibdo, suffix, null);
181     }
182 
183     @Nullable
184     public String makeWayForIncomingFile(final IBaseDataObject ibdo, final String suffix, @Nullable final String spec) {
185         // Construct output file from DropOff output path and IBaseDataObject filename
186         final String outputFile = getShortOutputFileName(ibdo, spec);
187 
188         // Return value when finally set
189         final String fileName = outputFile + suffix;
190 
191         // Set up path and make it writable
192         if (!setupPath(fileName)) {
193             return null;
194         }
195 
196         if (removeExistingFile(fileName)) {
197             return fileName;
198         }
199 
200         return null;
201     }
202 
203     /**
204      * Remove a file if it exists
205      */
206     public boolean removeExistingFile(final String fileName) {
207         // If a file already exists under this name, we're going
208         // to have to move it aside. This is going to break the link
209         // if it is an attachment to another parent document.
210         final Path theFile = Paths.get(fileName);
211         try {
212             Files.deleteIfExists(theFile);
213         } catch (IOException e) {
214             logger.error("Trouble removing file: {}", theFile, e);
215         }
216         return !Files.exists(theFile);
217     }
218 
219     /**
220      * mkdir -p to a path and make sure it is writable
221      *
222      * @param fileName the file name, including directory and filename parts
223      * @return true iff it works
224      */
225     public boolean setupPath(final String fileName) {
226         final String pathName = fileName.substring(0, fileName.lastIndexOf(SEPARATOR));
227         final Path thePath = Paths.get(pathName);
228 
229         // If the specified output directory doesn't exist try creating it
230         if (!Files.exists(thePath)) {
231             int tryCount = 1;
232             do {
233                 try {
234                     Files.createDirectories(thePath);
235                 } catch (IOException e) {
236                     logger.warn("Trouble setting up directories:{}", thePath, e);
237                 }
238                 if (!Files.exists(thePath)) {
239                     try {
240                         Thread.sleep(50L * tryCount);
241                     } catch (InterruptedException e) {
242                         Thread.currentThread().interrupt();
243                     }
244                 }
245                 tryCount++;
246             } while (!Files.exists(thePath) && tryCount <= 10);
247 
248             if (!Files.exists(thePath)) {
249                 logger.warn("Cannot create directory for output: {} in {} attempts", thePath, tryCount);
250                 return false;
251             }
252 
253             if (tryCount > 2 && logger.isDebugEnabled()) {
254                 logger.debug("Output path created for {} but it took {} attempts", thePath, tryCount);
255             }
256         }
257 
258         // If the specified output directory doesn't have write permission try to fix it
259         if (!Files.isWritable(thePath)) {
260             logger.warn("No write permission for {}, setting it now", pathName);
261             if (!thePath.toFile().setWritable(true) || !Files.isWritable(thePath)) {
262                 logger.warn("Cannot write to directory for output: {}", thePath);
263                 return false;
264             }
265         }
266 
267         return true;
268     }
269 
270     public String getOutputDirectory() {
271         return this.placeOutputData;
272     }
273 
274     /**
275      * Create a path name from the spec for the specified spec using d as the TLD if d is a TLD, null otherwise
276      *
277      * @param spec the spec to fill in
278      * @param d the payload to pull values from
279      * @see #getPathFromSpec(String,IBaseDataObject,IBaseDataObject)
280      */
281     public String getPathFromSpec(final String spec, final IBaseDataObject d) {
282         // pass self as tld if tld, null if not
283         return getPathFromSpec(spec, d, (d != null && !d.shortName().contains(Family.SEP)) ? d : null);
284     }
285 
286     /**
287      * Create a path name from the spec for the specified spec Specs understand the following 'language' of replacement
288      * stuff. Anything not understood is a literal
289      *
290      * <pre>
291      * <code>@TLD{'KEY'}</code> is to pull the named KEY from the top level document Metadata
292      * <code>@META{'KEY'}</code> is to pull the named KEY from the MetaData
293      * %U% = USER
294      * %I% = INPUT_FILE_NAME (whole thing)
295      * %S% = INPUT_FILE SHORT NAME
296      * %P% = INPUT_FILE PATH (all but short name)
297      * %i% = INPUT_FILE_NAME with slashes converted to underscores
298      * %p% = INPUT_FILE PATH with slashes converted to underscores
299      * %F% = FILETYPE
300      * %L% = LANGUAGE
301      * %G% = DTG multi directory layout yyyy-mm-dd/hh/mi(div)10
302      * %R% = ROOT (Unix or Win depending on OS)
303      * %B% = ID for the payload depending on type (no -att-)
304      * %b% = ID for the payload depending on type (with -att-)
305      * %Y% = Four digit year
306      * %M% = Two digit month
307      * %D% = Two digit day of month
308      * %J% = Three digit ordinal day of the year
309      * </pre>
310      *
311      * @param specArg the incoming specification
312      * @param d the payload we are making a path for
313      * @param tld the top level document in the d family, possibly null
314      * @return string path name with correct separators for this OS
315      */
316     public String getPathFromSpec(final String specArg, @Nullable final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
317         final StringBuilder sb = new StringBuilder(128);
318 
319         // Provide a default spec, just like the old days...
320         String spec = specArg;
321         if (spec == null) {
322             spec = "%R%/@TLD{'TARGETBIN'}/%S%";
323         }
324 
325         for (int i = 0; i < spec.length(); i++) {
326             final char c = spec.charAt(i);
327 
328             if (c == '%' && i < spec.length() - 2) {
329                 final char t = spec.charAt(i + 1);
330                 final char x = spec.charAt(i + 2);
331 
332                 if (x == c) {
333                     switch (t) {
334                         case 'U':
335                             if (tld != null) {
336                                 sb.append(nvl(tld.getParameter("UserName"), "no-userid"));
337                             } else if (d != null) {
338                                 sb.append(nvl(d.getParameter("UserName"), "no-userid"));
339                             }
340                             break;
341                         case 'S':
342                             if (d != null) {
343                                 sb.append(d.shortName());
344                             }
345                             break;
346                         case 'I':
347                             if (d != null) {
348                                 sb.append(d.getFilename());
349                             }
350                             break;
351                         case 'i':
352                             if (d != null) {
353                                 sb.append(d.getFilename().replaceAll("[/\\\\]", "_"));
354                             }
355                             break;
356                         case 'P':
357                             if (d != null) {
358                                 sb.append(d.getFilename(), 0, d.getFilename().length() - d.shortName().length());
359                             }
360                             break;
361                         case 'p':
362                             if (d != null) {
363                                 sb.append(d.getFilename().substring(0, d.getFilename().length() - d.shortName().length()).replaceAll("[/\\\\]", "_"));
364                             }
365                             break;
366                         case 'F':
367                             if (d != null) {
368                                 sb.append(nvl(cleanSpecPath(d.getFileType()), "NONE"));
369                             }
370                             break;
371                         case 'L':
372                             if (d != null) {
373                                 sb.append(nvl(d.getParameter("LANGUAGE"), "NONE"));
374                             }
375                             break;
376                         case 'G':
377                             if (tld != null) {
378                                 sb.append(datePath(cleanSpecPath(tld.getStringParameter("DTG"))));
379                             } else if (d != null) {
380                                 sb.append(datePath(cleanSpecPath(d.getStringParameter("DTG"))));
381                             }
382                             break;
383                         case 'R':
384                             sb.append(getRootPath());
385                             break;
386                         case 'B':
387                             if (tld != null) {
388                                 sb.append(cleanSpecPath(getBestIdFrom(tld)));
389                             } else if (d != null) {
390                                 sb.append(cleanSpecPath(getBestIdFrom(d)));
391                             }
392                             break;
393                         case 'b':
394                             sb.append(cleanSpecPath((tld != null) ? getBestIdFrom(tld) : getBestIdFrom(d)));
395                             final String sn = d.shortName();
396                             final int pos = sn.indexOf(Family.SEP);
397                             if (pos > 0) {
398                                 sb.append(sn.substring(pos));
399                             }
400                             break;
401                         case 'Y':
402                             sb.append(TimeUtil.getDate("yyyy", "GMT"));
403                             break;
404                         case 'M':
405                             sb.append(TimeUtil.getDate("MM", "GMT"));
406                             break;
407                         case 'D':
408                             sb.append(TimeUtil.getDate("dd", "GMT"));
409                             break;
410                         case 'J':
411                             sb.append(TimeUtil.getDate("DDD", "GMT"));
412                             break;
413                         default:
414                             sb.append(c).append(t).append(x);
415                     }
416                     i += 2; // SUPPRESS CHECKSTYLE ModifiedControlVariable
417                 } else {
418                     // No trailing % after character
419                     sb.append(c);
420                 }
421 
422             } else if (c == '@' && i < spec.length() - 8 && spec.charAt(i + 1) == 'M' && spec.charAt(i + 2) == 'E' && spec.charAt(i + 3) == 'T'
423                     && spec.charAt(i + 4) == 'A' && spec.charAt(i + 5) == '{' && spec.charAt(i + 6) == '\'') {
424 
425                 final int endpos = spec.indexOf("'", i + 7);
426                 if (endpos > i + 7) {
427                     final String token = spec.substring(i + 7, endpos);
428                     final String value = cleanSpecPath(d.getStringParameter(token));
429                     sb.append(nvl(value, "NO-" + token));
430                     i += 8 + token.length(); // META{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
431                 } else {
432                     sb.append(c);
433                 }
434             } else if (c == '@' && i < spec.length() - 7 && spec.charAt(i + 1) == 'T' && spec.charAt(i + 2) == 'L' && spec.charAt(i + 3) == 'D'
435                     && spec.charAt(i + 4) == '{' && spec.charAt(i + 5) == '\'' && tld != null) {
436                 final int endpos = spec.indexOf("'", i + 6);
437                 if (endpos > i + 6) {
438                     final String token = spec.substring(i + 6, endpos);
439                     final String value = cleanSpecPath(tld.getStringParameter(token));
440                     sb.append(nvl(value, "NO-" + token));
441                     i += 7 + token.length(); // TLD{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
442                 } else {
443                     sb.append(c);
444                 }
445             } else {
446                 sb.append(c);
447             }
448 
449         }
450 
451         String answer = sb.toString();
452 
453         // Set the proper path separator
454         answer = answer.replace('\\', '/');
455         answer = answer.replaceAll("\\.([/\\\\])", "_$1");
456 
457         return answer;
458     }
459 
460     @Nullable
461     protected String cleanSpecPath(@Nullable String token) {
462         return token == null ? null : token.replaceAll("[.]+", ".");
463     }
464 
465     /**
466      * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
467      * shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
468      * generated id.
469      *
470      * @return id
471      */
472     public String getBestIdFrom(final IBaseDataObject d) {
473         for (final String s : this.idTokens) {
474             if (!StringUtils.isBlank(d.getStringParameter(s))) {
475                 return d.getStringParameter(s);
476             }
477             if (SHORTNAME.equals(s)) {
478                 final String shortName = d.shortName();
479                 // if shortname is not blank use it, if blank move on...
480                 if (!StringUtils.isBlank(shortName)) {
481                     return shortName;
482                 }
483             }
484             if (AUTO_GENERATED_ID.equals(s)) {
485                 return getRandomUuid(d);
486             }
487         }
488         // if nothing else worked, use an auto gen id
489         return getRandomUuid(d);
490     }
491 
492     /**
493      * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
494      * shortname. An ID = AUTO_GENERATED_ID will be ignored. If no id value is found, returns empty array.
495      *
496      * @return id
497      * @deprecated use {@link #getExistingIdsList(IBaseDataObject)}
498      */
499     @Deprecated
500     @SuppressWarnings("AvoidObjectArrays")
501     public String[] getExistingIds(final IBaseDataObject d) {
502         return getExistingIdsList(d).toArray(new String[0]);
503     }
504 
505     /**
506      * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
507      * shortname. An ID = AUTO_GENERATED_ID will be ignored. If no id value is found, returns empty list.
508      *
509      * @return a list of id values
510      */
511     public List<String> getExistingIdsList(final IBaseDataObject d) {
512         final List<String> values = new ArrayList<>();
513         for (final String s : this.idTokens) {
514             if (!StringUtils.isBlank(d.getStringParameter(s))) {
515                 values.add(d.getStringParameter(s));
516             }
517             if (SHORTNAME.equals(s)) {
518                 final String shortName = d.shortName();
519                 // if shortname is not blank use it, if blank move on...
520                 if (!StringUtils.isBlank(shortName)) {
521                     values.add(shortName);
522                 }
523             }
524         }
525         return values;
526     }
527 
528     public String getRootPath() {
529         return this.unixRoot;
530     }
531 
532     public String getSubDirName(final IBaseDataObject d) {
533         return getSubDirName(d, null, null);
534     }
535 
536     public String getSubDirName(final IBaseDataObject d, @Nullable final String spec, @Nullable final IBaseDataObject tld) {
537         String fileName = null;
538 
539         if (StringUtils.isNotEmpty(spec)) {
540             fileName = getPathFromSpec(spec, d);
541         }
542 
543         if (StringUtils.isNotEmpty(fileName)) {
544             logger.debug("usingPathFromSpec instead of TARGETBIN: {}", fileName);
545             return fileName;
546         } else if (tld != null && tld.getStringParameter(TARGETBIN) != null) {
547             logger.debug("TARGETBIN is {}", tld.getParameter(TARGETBIN));
548             return fixFileNameSeparators(tld.getStringParameter(TARGETBIN));
549         } else {
550             logger.debug("TARGETBIN is null");
551             return ("NO-CASE" + SEPARATOR + TimeUtil.getCurrentDate());
552         }
553     }
554 
555     public String getRelativeShortOutputFileName(final IBaseDataObject d) {
556         return getRelativeShortOutputFileName(d, null);
557     }
558 
559     public String getRelativeShortOutputFileName(final IBaseDataObject d, @Nullable final String spec) {
560         return getRelativeShortOutputFileName(d, spec, !d.shortName().contains(Family.SEP) ? d : null);
561     }
562 
563     public String getRelativeShortOutputFileName(final IBaseDataObject d, final String spec, @Nullable final IBaseDataObject tld) {
564         final String sdir = getSubDirName(d, spec, tld);
565         if ("".equals(sdir)) {
566             return SEPARATOR + d.shortName();
567         } else {
568             return SEPARATOR + sdir + SEPARATOR + d.shortName();
569         }
570     }
571 
572     public String getShortOutputFileName(final IBaseDataObject d) {
573         return getShortOutputFileName(d, null);
574     }
575 
576     public String getShortOutputFileName(final IBaseDataObject d, @Nullable final String spec) {
577         return getShortOutputFileName(d, spec, !d.shortName().contains(Family.SEP) ? d : null);
578     }
579 
580     public String getShortOutputFileName(final IBaseDataObject d, final String spec, @Nullable final IBaseDataObject tld) {
581         return getRelativeShortOutputFileName(d, spec, tld);
582     }
583 
584     /**
585      * Replace any file separators that are not for this platform with the correct one
586      */
587     public String fixFileNameSeparators(final String s) {
588         // other platform;
589         String badfs;
590 
591         if ("/".equals(SEPARATOR)) {
592             badfs = "\\";
593         } else {
594             badfs = "/";
595         }
596 
597         final StringBuilder ret = new StringBuilder(s.length());
598 
599         for (int i = 0; i < s.length(); i++) {
600             if (s.charAt(i) == badfs.charAt(0)) {
601                 ret.append(SEPARATOR);
602             } else {
603                 ret.append(s.charAt(i));
604             }
605         }
606 
607         return ret.toString();
608     }
609 
610     protected Object nvl(@Nullable final Object a, final Object b) {
611         if (a != null) {
612             return a;
613         }
614         return b;
615     }
616 
617     /**
618      * Format string to date path (yyyy-mm-dd/hh/(mm%10))
619      *
620      * @param dtg expected format yyyymmddhhmmss
621      * @return yyyy-mm-dd/hh/(mm%10)
622      */
623     protected String datePath(@Nullable final String dtg) {
624         if (dtg == null) {
625             return TimeUtil.getDateAsPath(Instant.now());
626         } else {
627             return dtg.substring(0, 4) + "-" + // yyyy
628                     dtg.substring(4, 6) + "-" + // mm
629                     dtg.substring(6, 8) + "/" + // dd
630                     dtg.substring(8, 10) + "/" + // HH
631                     dtg.substring(10, 11) + "0"; // M0
632         }
633     }
634 
635     /**
636      * Make a dot file name from the supplied path Ex: /path/to/file.txt --&gt; /path/to/.file.txt
637      *
638      * @param fullName the full path and filename of the file
639      * @return the path as is but with a leading dot in the filename
640      */
641     public String makeDotFile(final String fullName) {
642         final int fpos = fullName.lastIndexOf("/");
643         final int rpos = fullName.lastIndexOf("\\");
644         if (fpos == -1 && rpos == -1) {
645             return "." + fullName;
646         }
647         final int pos = Math.max(fpos, rpos);
648         return fullName.substring(0, pos + 1) + "." + fullName.substring(pos + 1);
649     }
650 
651     /**
652      * Get the file type from the metadata or the form string passed in
653      *
654      * @param bdo IBaseDataObject
655      * @return the file type
656      */
657     public static String getFileType(final IBaseDataObject bdo) {
658         return getAndPutFileType(bdo, null, null);
659     }
660 
661     /**
662      * Get the file type from the IBaseDataObject or the form string passed in
663      *
664      * @param bdo IBaseDataObject
665      * @param metaData Optional map of metadata that might be modified.
666      * @param formsArg Optional space separated string of current forms
667      * @return the file type
668      */
669     public static String getAndPutFileType(final IBaseDataObject bdo, @Nullable final Map<String, String> metaData, @Nullable final String formsArg) {
670         String forms = formsArg;
671         if (forms == null) {
672             forms = bdo.getStringParameter(FileTypeCheckParameter.POPPED_FORMS.getFieldName());
673             if (forms == null) {
674                 forms = "";
675             }
676         }
677 
678         String fileType;
679         if (bdo.hasParameter(FileTypeCheckParameter.FILETYPE.getFieldName())) {
680             fileType = bdo.getStringParameter(FileTypeCheckParameter.FILETYPE.getFieldName());
681         } else if (bdo.hasParameter(FileTypeCheckParameter.FINAL_ID.getFieldName())) {
682             fileType = bdo.getStringParameter(FileTypeCheckParameter.FINAL_ID.getFieldName());
683             logger.debug("FINAL_ID FileType is ({})", fileType);
684             if (metaData != null) {
685                 metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
686             }
687         } else {
688             if (forms.contains(" ")) {
689                 fileType = forms.substring(0, forms.indexOf(" ")).trim();
690                 if (metaData != null) {
691                     metaData.put(FileTypeCheckParameter.COMPLETE_FILETYPE.getFieldName(), forms);
692                 }
693             } else {
694                 fileType = forms;
695             }
696             if (StringUtils.isEmpty(fileType)) {
697                 if (bdo.hasParameter(FileTypeCheckParameter.FONT_ENCODING.getFieldName())) {
698                     fileType = TEXT;
699                 } else {
700                     fileType = UNKNOWN;
701                 }
702             }
703 
704             if (metaData != null) {
705                 metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
706             }
707         }
708 
709         if (UNKNOWN.equals(fileType) && forms.contains("MSWORD")) {
710             fileType = "MSWORD_FRAGMENT";
711         }
712 
713         if ("QUOTED-PRINTABLE".equals(fileType) || fileType.startsWith(PREFIXES_LANG) || fileType.startsWith("ENCODING(")) {
714             fileType = TEXT;
715         }
716 
717         return fileType;
718     }
719 
720     /**
721      * Parameters that are used to determine filetype in {@link #getAndPutFileType(IBaseDataObject, Map, String)}
722      */
723     public enum FileTypeCheckParameter {
724         COMPLETE_FILETYPE("COMPLETE_FILETYPE"), FILETYPE("FILETYPE"), FINAL_ID("FINAL_ID"), FONT_ENCODING("FontEncoding"), POPPED_FORMS(
725                 "POPPED_FORMS");
726 
727         final String fieldName;
728 
729         FileTypeCheckParameter(String fieldName) {
730             this.fieldName = fieldName;
731         }
732 
733         public String getFieldName() {
734             return fieldName;
735         }
736     }
737 
738     /**
739      * Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
740      * shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
741      * generated id.
742      *
743      * @param d the payload
744      * @param tld if a param is specified and cannot be found in the d method parameter, tries to get param from tld.
745      * @return the id based the best ID available, shortname or auto gen id. If no id is found, defaults to auto gen.
746      */
747     public String getBestId(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
748 
749         for (final String s : this.idTokens) {
750             if (AUTO_GENERATED_ID.equals(s)) {
751                 String parentAutoGeneratedId = null;
752                 if (tld != null) {
753                     parentAutoGeneratedId = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
754                 }
755                 if (StringUtils.isBlank(parentAutoGeneratedId)) {
756                     String uuid = getRandomUuid(d);
757                     if (tld != null) {
758                         tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
759                     }
760                     if (!StringUtils.isBlank(uuid)) {
761                         final String component = d.shortName();
762                         final int pos = component.indexOf(Family.SEP);
763                         if (pos > -1) {
764                             uuid += component.substring(pos);
765                         }
766                         return uuid;
767                     }
768                 }
769                 String uuid = null;
770                 // try getting param from the tld
771                 if (!StringUtils.isBlank(parentAutoGeneratedId) && (tld != null)) {
772                     uuid = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
773                 }
774                 if (!StringUtils.isBlank(uuid)) {
775                     final String component = d.shortName();
776                     final int pos = component.indexOf(Family.SEP);
777                     if (pos > -1) {
778                         uuid += component.substring(pos);
779                     }
780                     d.setParameter(AUTO_GENERATED_ID, "yes");
781                     return uuid;
782                 }
783 
784             }
785 
786             if (SHORTNAME.equals(s)) {
787                 final String shortName = d.shortName();
788 
789                 // if shortname is not blank use it, if blank move on...
790                 if (!StringUtils.isBlank(shortName)) {
791                     return shortName;
792                 }
793 
794                 String path = d.getStringParameter(s);
795                 // try getting shortname from the tld
796                 if (StringUtils.isBlank(path) && (tld != null)) {
797                     path = tld.getStringParameter(s);
798                 }
799                 if (!StringUtils.isBlank(path)) {
800                     final String component = d.shortName();
801                     final int pos = component.indexOf(Family.SEP);
802                     if (pos > -1) {
803                         path += component.substring(pos);
804                     }
805                     return path;
806                 }
807 
808             }
809             // the param from the tld has priority over any child
810             if ((tld != null) && !StringUtils.isBlank(tld.getStringParameter(s))) {
811                 String path = tld.getStringParameter(s);
812                 if (!StringUtils.isBlank(path)) {
813                     final String component = d.shortName();
814                     final int pos = component.indexOf(Family.SEP);
815                     if (pos > -1) {
816                         path += component.substring(pos);
817                     }
818                     return path;
819                 }
820             }
821             // if the param is not in the tld
822             if (!StringUtils.isBlank(d.getStringParameter(s))) {
823                 return d.getStringParameter(s);
824             }
825 
826         }
827 
828         // if nothing works, auto gen an id
829         final String uuid = getRandomUuid(d);
830         if (tld != null) {
831             tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
832         }
833         return uuid;
834     }
835 
836     /**
837      * Creates a UUID. Includes a prefix if specified.
838      *
839      * @param d Adds a parameter indicating an auto gen id.
840      * @return uuid
841      */
842     private String getRandomUuid(final IBaseDataObject d) {
843         String uuid = UUID.randomUUID().toString();
844         // use the prefix
845         if (!StringUtils.isBlank(this.autoGeneratedIdPrefix)) {
846             uuid = uuid.substring(this.autoGeneratedIdPrefix.length());
847             uuid = this.autoGeneratedIdPrefix + uuid;
848         }
849         d.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
850         d.setParameter(AUTO_GENERATED_ID, "yes");
851         return uuid;
852     }
853 
854     /**
855      * Extract language or a default value
856      *
857      * @param d the payload
858      * @return the language or the default value (never null)
859      */
860     public String getLanguage(final IBaseDataObject d) {
861         String lang = d.getStringParameter("LANGUAGE");
862         if (lang == null) {
863             lang = "NONE";
864         }
865         return lang;
866     }
867 
868     /**
869      * Handle data
870      *
871      * @param d the data object in question
872      * @return the Date when the event occurred
873      */
874     public Date getEventDate(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
875         Date eventDate = extractEventDateFrom(d, false);
876         if (eventDate == null && tld != null) {
877             eventDate = extractEventDateFrom(tld, this.defaultEventDateToNow);
878         }
879         return eventDate;
880     }
881 
882     @Nullable
883     public Date extractEventDateFrom(final IBaseDataObject d, final boolean lastResortDefault) {
884         for (final String paramName : this.dateTokens) {
885             final String value = d.getStringParameter(paramName);
886             if (value != null) {
887                 try {
888                     ZonedDateTime zdt = FlexibleDateTimeParser.parse(value, DATE_ISO_8601);
889                     if (zdt == null) {
890                         logger.debug("FlexibleDateTimeParser returned null trying to parse EventDate");
891                     } else {
892                         return Date.from(zdt.toInstant());
893                     }
894                 } catch (DateTimeParseException ex) {
895                     logger.debug("Cannot parse EventDate", ex);
896                 }
897             }
898         }
899 
900         // Default to current system time if last resort
901         return lastResortDefault ? Date.from(Instant.now()) : null;
902     }
903 
904     /**
905      * Utilizes the static methods getFullFilepathsFromParams and getFileExtensions to extract the file extensions from all
906      * the filenames of the object of a given {@link IBaseDataObject}. If one or more file extensions are extracted, the
907      * IBaseDataObject's FILEXT parameter is set as the unique set of extracted file extensions, converted to lowercase.
908      *
909      * @param p IBaseDataObject to process
910      *
911      */
912     void extractUniqueFileExtensions(IBaseDataObject p) {
913         List<String> filenames = getFullFilepathsFromParams(p);
914         Set<String> extensions = getFileExtensions(filenames, this.maxFilextLen);
915         if (!extensions.isEmpty()) {
916             p.setParameter(FILEXT, extensions);
917         }
918     }
919 
920     /**
921      * Given a list of filenames, extract and return a set of non-blank file extensions converted to lowercase.
922      *
923      * @param filenames The list of filenames to examine
924      * @param maxFilextLen The maximum size we want a file extension to be
925      * @return A set of unique file extensions from the filename list
926      */
927     public static Set<String> getFileExtensions(List<String> filenames, int maxFilextLen) {
928         final Set<String> extensions = new HashSet<>();
929         for (String filename : filenames) {
930 
931             // add the file extension if it is smaller than maxFilextLen
932             final String fext = FilenameUtils.getExtension(filename);
933             if (StringUtils.isNotBlank(fext) && fext.length() <= maxFilextLen) {
934                 extensions.add(fext.toLowerCase(Locale.getDefault()));
935             }
936         }
937         return extensions;
938     }
939 
940     /**
941      * Checks the Original-Filename and FILE_ABSOLUTEPATH for the filename of the object. Returns a list with the non-empty
942      * strings found in these fields. If nothing is found in either field, return an empty list.
943      *
944      * @param d The IBDO
945      * @return The list of filenames found in the field Original-Filename or FILE_ABSOLUTEPATH
946      */
947     public static List<String> getFullFilepathsFromParams(IBaseDataObject d) {
948         return getFullFilepathsFromParams(d, new String[] {ORIGINAL_FILENAME, FILE_ABSOLUTEPATH});
949     }
950 
951     /**
952      * Uses the specified list of fields to check for filenames of the object. Returns a list with the non-empty strings
953      * found in these fields. If nothing is found in either field, return an empty list.
954      *
955      * @param d The IBDO
956      * @param filenameFields The list of fields on the IBDO to check
957      * @return The list of filenames found in the list of fields on the IBDO
958      * @deprecated use {@link #getFullFilepathsFromParams(IBaseDataObject, List)}
959      */
960     @Deprecated
961     @SuppressWarnings("AvoidObjectArrays")
962     public static List<String> getFullFilepathsFromParams(IBaseDataObject d, String[] filenameFields) {
963         return getFullFilepathsFromParams(d, Arrays.asList(filenameFields));
964     }
965 
966     /**
967      * Uses the specified list of fields to check for filenames of the object. Returns a list with the non-empty strings
968      * found in these fields. If nothing is found in either field, return an empty list.
969      *
970      * @param d The IBDO
971      * @param filenameFields The list of fields on the IBDO to check
972      * @return The list of filenames found in the list of fields on the IBDO
973      */
974     public static List<String> getFullFilepathsFromParams(IBaseDataObject d, List<String> filenameFields) {
975 
976         List<String> filenames = new ArrayList<>();
977 
978         for (String ibdoField : filenameFields) {
979             if (d.hasParameter(ibdoField)) {
980                 for (Object filename : d.getParameter(ibdoField)) {
981                     String stringFileName = (String) filename;
982                     if (StringUtils.isNotBlank(stringFileName)) {
983                         filenames.add(stringFileName);
984                     }
985                 }
986             }
987         }
988         return filenames;
989     }
990 
991     /**
992      * Process metadata before doing any output
993      *
994      * @param payloadList list of items being dropped off that may need initial metadata computations
995      */
996     public void processMetadata(final List<IBaseDataObject> payloadList) {
997 
998         // Keep track of parent's filetype to output; relies on the attachments being sorted
999         final Map<String, String> parentTypes = new HashMap<>();
1000         final IBaseDataObject tld = payloadList.get(0);
1001         final List<String> extendedFileTypes = new ArrayList<>();
1002         parentTypes.put("1", tld.getFileType());
1003         for (int i = 0; i < parentParams.size(); i++) {
1004             final String param = parentParams.get(i);
1005             if (tld.hasParameter(param)) {
1006                 parentTypes.put("1" + param, tld.getStringParameter(param));
1007             }
1008         }
1009 
1010         for (final IBaseDataObject p : payloadList) {
1011             final int level = StringUtils.countMatches(p.shortName(), Family.SEP) + 1;
1012             // save specified metadata items for children to grab
1013             parentTypes.put("" + level, p.getFileType());
1014 
1015             extractUniqueFileExtensions(p);
1016 
1017             if (p.getStringParameter(EXTENDED_FILETYPE) == null) {
1018                 extendedFileTypes.clear();
1019                 for (final Map.Entry<String, Collection<Object>> entry : p.getParameters().entrySet()) {
1020                     final String key = entry.getKey();
1021                     if (key != null && key.endsWith("_FILETYPE")) {
1022                         for (final Object value : entry.getValue()) {
1023                             final String vs = value.toString();
1024                             if (!extendedFileTypes.contains(vs)) {
1025                                 extendedFileTypes.add(vs);
1026                             }
1027                         }
1028                     }
1029                 }
1030                 if (!extendedFileTypes.isEmpty()) {
1031                     final StringBuilder extft = new StringBuilder(getFileType(p));
1032                     for (int j = 0; j < extendedFileTypes.size(); j++) {
1033                         final String s = extendedFileTypes.get(j);
1034                         extft.append("//").append(s);
1035                     }
1036                     p.setParameter(EXTENDED_FILETYPE, extft.toString());
1037                 }
1038             }
1039 
1040             for (int j = 0; j < parentParams.size(); j++) {
1041                 final String param = parentParams.get(j);
1042                 if (p.hasParameter(param)) {
1043                     parentTypes.put("" + level + param, p.getStringParameter(param));
1044                 } else {
1045                     // Must clear to keep my children from getting their uncle's value
1046                     parentTypes.remove("" + level + param);
1047                 }
1048 
1049             }
1050 
1051             if (level > 1) {
1052                 // then it might want some PARENT info
1053 
1054                 final int parentLevel = level - 1;
1055                 final String pType = parentTypes.get("" + parentLevel);
1056                 if (StringUtils.isNotEmpty(pType)) {
1057                     p.setParameter(PARENT_FILETYPE, pType);
1058                 } else {
1059                     p.setParameter(PARENT_FILETYPE, parentTypes.get("1"));
1060                 }
1061                 for (int j = 0; j < parentParams.size(); j++) {
1062                     final String param = parentParams.get(j);
1063                     int plvl = parentLevel;
1064                     while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
1065                         plvl--;
1066                     }
1067                     if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
1068                         p.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
1069                     }
1070                 }
1071             }
1072 
1073             if (p.hasExtractedRecords()) {
1074                 final List<IBaseDataObject> childObjList = p.getExtractedRecords();
1075                 childObjList.sort(new ShortNameComparator());
1076                 for (final IBaseDataObject child : childObjList) {
1077                     final int parentLevel = StringUtils.countMatches(child.shortName(), Family.SEP);
1078                     final String parentFileType = parentTypes.get("" + parentLevel);
1079                     if (parentFileType != null) {
1080                         child.setParameter(PARENT_FILETYPE, parentFileType);
1081                     }
1082                     for (int k = 0; k < parentParams.size(); k++) {
1083                         final String param = parentParams.get(k);
1084                         int plvl = parentLevel;
1085                         while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
1086                             plvl--;
1087                         }
1088                         if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
1089                             child.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
1090                         }
1091                     }
1092                 }
1093             }
1094         }
1095     }
1096 }