DropOffUtil.java
package emissary.output;
import emissary.config.ConfigUtil;
import emissary.config.Configurator;
import emissary.core.Family;
import emissary.core.IBaseDataObject;
import emissary.util.FlexibleDateTimeParser;
import emissary.util.ShortNameComparator;
import emissary.util.TimeUtil;
import emissary.util.shell.Executrix;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import javax.annotation.Nullable;
import static emissary.core.Form.PREFIXES_LANG;
import static emissary.core.Form.TEXT;
import static emissary.core.Form.UNKNOWN;
import static emissary.core.constants.Parameters.FILEXT;
import static emissary.core.constants.Parameters.FILE_ABSOLUTEPATH;
import static emissary.core.constants.Parameters.ORIGINAL_FILENAME;
import static emissary.util.TimeUtil.DATE_ISO_8601;
import static emissary.util.TimeUtil.getDateOrdinalWithTime;
public class DropOffUtil {
protected static final Logger logger = LoggerFactory.getLogger(DropOffUtil.class);
protected static final String SEPARATOR = FileSystems.getDefault().getSeparator();
protected static final String OS_NAME = System.getProperty("os.name").toUpperCase(Locale.getDefault());
protected String unixRoot;
protected String placeOutputData;
/** Sources for building an ID for an item */
protected List<String> idTokens = new ArrayList<>();
/** Sources for building a date string for an item */
protected List<String> dateTokens = new ArrayList<>();
/** params to get from parent and save as PARENT_param */
protected List<String> parentParams = new ArrayList<>();
/** Output directories */
protected Map<String, File> outputDirectories = new HashMap<>();
protected Executrix executrix;
// Items for generating random filenames
protected static final SecureRandom prng = new SecureRandom();
protected static final byte[] ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes();
protected String prefix = "TXT";
protected boolean uuidInOutputFilenames = true;
protected int maxFilextLen = Integer.MAX_VALUE;
// Items for generating UUIDs
private static final String AUTO_GENERATED_ID = "AUTO_GENERATED_ID";
private static final String PARENT_AUTO_GENERATED_ID = "PARENT_AUTO_GENERATED_ID";
@Nullable
private String autoGeneratedIdPrefix = null;
protected static final String EXTENDED_FILETYPE = "EXTENDED_FILETYPE";
protected static final String PARENT_FILETYPE = "PARENT_FILETYPE";
protected static final String SHORTNAME = "SHORTNAME";
protected static final String TARGETBIN = "TARGETBIN";
private static final String DEFAULT_EVENT_DATE_TO_NOW = "DEFAULT_EVENT_DATE_TO_NOW";
protected boolean defaultEventDateToNow = true;
/**
* Create with the default configuration
*/
public DropOffUtil() {
configure(null);
}
/**
* Create with the specified configuration
*/
public DropOffUtil(final Configurator configG) {
configure(configG);
}
/**
* Set up config for this class
* <ul>
* <li>ID_PARAMETER : multiple parameter values, ordered list of how to build a EMISSARY_ID</li>
* <li>ID : backwards compatibility for ID_PARAMETER only used if ID_PARAMETER does not exist</li>
* <li>DATE_PARAMETER : multiple parameter values, ordered list of how to build a date path</li>
* <li>OUTPUT_FILE_PREFIX: string to use when generating random filenames, default: TXT</li>
* <li>UUID_IN_OUTPUT_FILENAMES: boolean [true]</li>
* <li>AUTO_GENERATED_ID_PREFIX: prefix to use for an auto-generated id</li>
* </ul>
*/
protected void configure(@Nullable final Configurator configG) {
Configurator actualConfigG = configG;
if (actualConfigG == null) {
try {
actualConfigG = ConfigUtil.getConfigInfo(DropOffUtil.class);
} catch (IOException e) {
logger.error("Cannot open default config file", e);
}
}
if (actualConfigG != null) {
this.placeOutputData = actualConfigG.findStringEntry("OUTPUT_DATA", "outputData");
this.unixRoot = actualConfigG.findStringEntry("UNIX_ROOT", null);
this.executrix = new Executrix(actualConfigG);
this.idTokens = actualConfigG.findEntries("ID_PARAMETER");
this.autoGeneratedIdPrefix = actualConfigG.findStringEntry("AUTO_GENERATED_ID_PREFIX");
// truncate the prefix if necessary
if (!StringUtils.isBlank(this.autoGeneratedIdPrefix) && (this.autoGeneratedIdPrefix.length() > 4)) {
this.autoGeneratedIdPrefix = this.autoGeneratedIdPrefix.substring(0, 4);
}
if (this.idTokens.isEmpty()) {
this.idTokens = actualConfigG.findEntries("ID");
}
this.dateTokens = actualConfigG.findEntries("DATE_PARAMETER");
this.parentParams = actualConfigG.findEntries("PARENT_PARAM");
this.defaultEventDateToNow = actualConfigG.findBooleanEntry(DEFAULT_EVENT_DATE_TO_NOW, this.defaultEventDateToNow);
this.prefix = actualConfigG.findStringEntry("OUTPUT_FILE_PREFIX", prefix);
this.uuidInOutputFilenames = actualConfigG.findBooleanEntry("UUID_IN_OUTPUT_FILENAMES", this.uuidInOutputFilenames);
// limit the length of the FILEXT param
this.maxFilextLen = actualConfigG.findIntEntry("MAX_FILEXT_LEN", this.maxFilextLen);
if (this.maxFilextLen < 0) {
this.maxFilextLen = Integer.MAX_VALUE;
}
} else {
logger.debug("Configuration is null for DropOffUtil, using defaults");
this.executrix = new Executrix();
}
}
/**
* Generate a new random build file name using the configured prefix and strategy
*/
public String generateBuildFileName() {
if (this.uuidInOutputFilenames) {
return (prefix + getDateOrdinalWithTime(Instant.now()) + UUID.randomUUID());
} else {
// Using some constants plus yyyyJJJhhmmss plus random digit, letter, digit
return (prefix + getDateOrdinalWithTime(Instant.now()) + prng.nextInt(10) + ALPHABET[prng.nextInt(ALPHABET.length)] + prng.nextInt(10));
}
}
/**
* Drop off often needs to make way for a file it wants to write Do so and return the name of the file that can now be
* written
*/
public String makeWayForIncomingFile(final IBaseDataObject ibdo, final String suffix) {
return makeWayForIncomingFile(ibdo, suffix, null);
}
@Nullable
public String makeWayForIncomingFile(final IBaseDataObject ibdo, final String suffix, @Nullable final String spec) {
// Construct output file from DropOff output path and IBaseDataObject filename
final String outputFile = getShortOutputFileName(ibdo, spec);
// Return value when finally set
final String fileName = outputFile + suffix;
// Set up path and make it writable
if (!setupPath(fileName)) {
return null;
}
if (removeExistingFile(fileName)) {
return fileName;
}
return null;
}
/**
* Remove a file if it exists
*/
public boolean removeExistingFile(final String fileName) {
// If a file already exists under this name, we're going
// to have to move it aside. This is going to break the link
// if it is an attachment to another parent document.
final Path theFile = Paths.get(fileName);
try {
Files.deleteIfExists(theFile);
} catch (IOException e) {
logger.error("Trouble removing file: {}", theFile, e);
}
return !Files.exists(theFile);
}
/**
* mkdir -p to a path and make sure it is writable
*
* @param fileName the file name, including directory and filename parts
* @return true iff it works
*/
public boolean setupPath(final String fileName) {
final String pathName = fileName.substring(0, fileName.lastIndexOf(SEPARATOR));
final Path thePath = Paths.get(pathName);
// If the specified output directory doesn't exist try creating it
if (!Files.exists(thePath)) {
int tryCount = 1;
do {
try {
Files.createDirectories(thePath);
} catch (IOException e) {
logger.warn("Trouble setting up directories:{}", thePath, e);
}
if (!Files.exists(thePath)) {
try {
Thread.sleep(50L * tryCount);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
tryCount++;
} while (!Files.exists(thePath) && tryCount <= 10);
if (!Files.exists(thePath)) {
logger.warn("Cannot create directory for output: {} in {} attempts", thePath, tryCount);
return false;
}
if (tryCount > 2 && logger.isDebugEnabled()) {
logger.debug("Output path created for {} but it took {} attempts", thePath, tryCount);
}
}
// If the specified output directory doesn't have write permission try to fix it
if (!Files.isWritable(thePath)) {
logger.warn("No write permission for {}, setting it now", pathName);
if (!thePath.toFile().setWritable(true) || !Files.isWritable(thePath)) {
logger.warn("Cannot write to directory for output: {}", thePath);
return false;
}
}
return true;
}
public String getOutputDirectory() {
return this.placeOutputData;
}
/**
* Create a path name from the spec for the specified spec using d as the TLD if d is a TLD, null otherwise
*
* @param spec the spec to fill in
* @param d the payload to pull values from
* @see #getPathFromSpec(String,IBaseDataObject,IBaseDataObject)
*/
public String getPathFromSpec(final String spec, final IBaseDataObject d) {
// pass self as tld if tld, null if not
return getPathFromSpec(spec, d, (d != null && !d.shortName().contains(Family.SEP)) ? d : null);
}
/**
* Create a path name from the spec for the specified spec Specs understand the following 'language' of replacement
* stuff. Anything not understood is a literal
*
* <pre>
* <code>@TLD{'KEY'}</code> is to pull the named KEY from the top level document Metadata
* <code>@META{'KEY'}</code> is to pull the named KEY from the MetaData
* %U% = USER
* %I% = INPUT_FILE_NAME (whole thing)
* %S% = INPUT_FILE SHORT NAME
* %P% = INPUT_FILE PATH (all but short name)
* %i% = INPUT_FILE_NAME with slashes converted to underscores
* %p% = INPUT_FILE PATH with slashes converted to underscores
* %F% = FILETYPE
* %L% = LANGUAGE
* %G% = DTG multi directory layout yyyy-mm-dd/hh/mi(div)10
* %R% = ROOT (Unix or Win depending on OS)
* %B% = ID for the payload depending on type (no -att-)
* %b% = ID for the payload depending on type (with -att-)
* %Y% = Four digit year
* %M% = Two digit month
* %D% = Two digit day of month
* %J% = Three digit ordinal day of the year
* </pre>
*
* @param specArg the incoming specification
* @param d the payload we are making a path for
* @param tld the top level document in the d family, possibly null
* @return string path name with correct separators for this OS
*/
public String getPathFromSpec(final String specArg, @Nullable final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
final StringBuilder sb = new StringBuilder(128);
// Provide a default spec, just like the old days...
String spec = specArg;
if (spec == null) {
spec = "%R%/@TLD{'TARGETBIN'}/%S%";
}
for (int i = 0; i < spec.length(); i++) {
final char c = spec.charAt(i);
if (c == '%' && i < spec.length() - 2) {
final char t = spec.charAt(i + 1);
final char x = spec.charAt(i + 2);
if (x == c) {
switch (t) {
case 'U':
if (tld != null) {
sb.append(nvl(tld.getParameter("UserName"), "no-userid"));
} else if (d != null) {
sb.append(nvl(d.getParameter("UserName"), "no-userid"));
}
break;
case 'S':
if (d != null) {
sb.append(d.shortName());
}
break;
case 'I':
if (d != null) {
sb.append(d.getFilename());
}
break;
case 'i':
if (d != null) {
sb.append(d.getFilename().replaceAll("[/\\\\]", "_"));
}
break;
case 'P':
if (d != null) {
sb.append(d.getFilename(), 0, d.getFilename().length() - d.shortName().length());
}
break;
case 'p':
if (d != null) {
sb.append(d.getFilename().substring(0, d.getFilename().length() - d.shortName().length()).replaceAll("[/\\\\]", "_"));
}
break;
case 'F':
if (d != null) {
sb.append(nvl(cleanSpecPath(d.getFileType()), "NONE"));
}
break;
case 'L':
if (d != null) {
sb.append(nvl(d.getParameter("LANGUAGE"), "NONE"));
}
break;
case 'G':
if (tld != null) {
sb.append(datePath(cleanSpecPath(tld.getStringParameter("DTG"))));
} else if (d != null) {
sb.append(datePath(cleanSpecPath(d.getStringParameter("DTG"))));
}
break;
case 'R':
sb.append(getRootPath());
break;
case 'B':
if (tld != null) {
sb.append(cleanSpecPath(getBestIdFrom(tld)));
} else if (d != null) {
sb.append(cleanSpecPath(getBestIdFrom(d)));
}
break;
case 'b':
sb.append(cleanSpecPath((tld != null) ? getBestIdFrom(tld) : getBestIdFrom(d)));
final String sn = d.shortName();
final int pos = sn.indexOf(Family.SEP);
if (pos > 0) {
sb.append(sn.substring(pos));
}
break;
case 'Y':
sb.append(TimeUtil.getDate("yyyy", "GMT"));
break;
case 'M':
sb.append(TimeUtil.getDate("MM", "GMT"));
break;
case 'D':
sb.append(TimeUtil.getDate("dd", "GMT"));
break;
case 'J':
sb.append(TimeUtil.getDate("DDD", "GMT"));
break;
default:
sb.append(c).append(t).append(x);
}
i += 2; // SUPPRESS CHECKSTYLE ModifiedControlVariable
} else {
// No trailing % after character
sb.append(c);
}
} else if (c == '@' && i < spec.length() - 8 && spec.charAt(i + 1) == 'M' && spec.charAt(i + 2) == 'E' && spec.charAt(i + 3) == 'T'
&& spec.charAt(i + 4) == 'A' && spec.charAt(i + 5) == '{' && spec.charAt(i + 6) == '\'') {
final int endpos = spec.indexOf("'", i + 7);
if (endpos > i + 7) {
final String token = spec.substring(i + 7, endpos);
final String value = cleanSpecPath(d.getStringParameter(token));
sb.append(nvl(value, "NO-" + token));
i += 8 + token.length(); // META{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
} else {
sb.append(c);
}
} else if (c == '@' && i < spec.length() - 7 && spec.charAt(i + 1) == 'T' && spec.charAt(i + 2) == 'L' && spec.charAt(i + 3) == 'D'
&& spec.charAt(i + 4) == '{' && spec.charAt(i + 5) == '\'' && tld != null) {
final int endpos = spec.indexOf("'", i + 6);
if (endpos > i + 6) {
final String token = spec.substring(i + 6, endpos);
final String value = cleanSpecPath(tld.getStringParameter(token));
sb.append(nvl(value, "NO-" + token));
i += 7 + token.length(); // TLD{'token'} SUPPRESS CHECKSTYLE ModifiedControlVariable
} else {
sb.append(c);
}
} else {
sb.append(c);
}
}
String answer = sb.toString();
// Set the proper path separator
answer = answer.replace('\\', '/');
answer = answer.replaceAll("\\.([/\\\\])", "_$1");
return answer;
}
@Nullable
protected String cleanSpecPath(@Nullable String token) {
return token == null ? null : token.replaceAll("[.]+", ".");
}
/**
* Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
* shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
* generated id.
*
* @return id
*/
public String getBestIdFrom(final IBaseDataObject d) {
for (final String s : this.idTokens) {
if (!StringUtils.isBlank(d.getStringParameter(s))) {
return d.getStringParameter(s);
}
if (SHORTNAME.equals(s)) {
final String shortName = d.shortName();
// if shortname is not blank use it, if blank move on...
if (!StringUtils.isBlank(shortName)) {
return shortName;
}
}
if (AUTO_GENERATED_ID.equals(s)) {
return getRandomUuid(d);
}
}
// if nothing else worked, use an auto gen id
return getRandomUuid(d);
}
/**
* Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
* shortname. An ID = AUTO_GENERATED_ID will be ignored. If no id value is found, returns empty array.
*
* @return id
* @deprecated use {@link #getExistingIdsList(IBaseDataObject)}
*/
@Deprecated
@SuppressWarnings("AvoidObjectArrays")
public String[] getExistingIds(final IBaseDataObject d) {
return getExistingIdsList(d).toArray(new String[0]);
}
/**
* Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
* shortname. An ID = AUTO_GENERATED_ID will be ignored. If no id value is found, returns empty list.
*
* @return a list of id values
*/
public List<String> getExistingIdsList(final IBaseDataObject d) {
final List<String> values = new ArrayList<>();
for (final String s : this.idTokens) {
if (!StringUtils.isBlank(d.getStringParameter(s))) {
values.add(d.getStringParameter(s));
}
if (SHORTNAME.equals(s)) {
final String shortName = d.shortName();
// if shortname is not blank use it, if blank move on...
if (!StringUtils.isBlank(shortName)) {
values.add(shortName);
}
}
}
return values;
}
public String getRootPath() {
return this.unixRoot;
}
public String getSubDirName(final IBaseDataObject d) {
return getSubDirName(d, null, null);
}
public String getSubDirName(final IBaseDataObject d, @Nullable final String spec, @Nullable final IBaseDataObject tld) {
String fileName = null;
if (StringUtils.isNotEmpty(spec)) {
fileName = getPathFromSpec(spec, d);
}
if (StringUtils.isNotEmpty(fileName)) {
logger.debug("usingPathFromSpec instead of TARGETBIN: {}", fileName);
return fileName;
} else if (tld != null && tld.getStringParameter(TARGETBIN) != null) {
logger.debug("TARGETBIN is {}", tld.getParameter(TARGETBIN));
return fixFileNameSeparators(tld.getStringParameter(TARGETBIN));
} else {
logger.debug("TARGETBIN is null");
return ("NO-CASE" + SEPARATOR + TimeUtil.getCurrentDate());
}
}
public String getRelativeShortOutputFileName(final IBaseDataObject d) {
return getRelativeShortOutputFileName(d, null);
}
public String getRelativeShortOutputFileName(final IBaseDataObject d, @Nullable final String spec) {
return getRelativeShortOutputFileName(d, spec, !d.shortName().contains(Family.SEP) ? d : null);
}
public String getRelativeShortOutputFileName(final IBaseDataObject d, final String spec, @Nullable final IBaseDataObject tld) {
final String sdir = getSubDirName(d, spec, tld);
if ("".equals(sdir)) {
return SEPARATOR + d.shortName();
} else {
return SEPARATOR + sdir + SEPARATOR + d.shortName();
}
}
public String getShortOutputFileName(final IBaseDataObject d) {
return getShortOutputFileName(d, null);
}
public String getShortOutputFileName(final IBaseDataObject d, @Nullable final String spec) {
return getShortOutputFileName(d, spec, !d.shortName().contains(Family.SEP) ? d : null);
}
public String getShortOutputFileName(final IBaseDataObject d, final String spec, @Nullable final IBaseDataObject tld) {
return getRelativeShortOutputFileName(d, spec, tld);
}
/**
* Replace any file separators that are not for this platform with the correct one
*/
public String fixFileNameSeparators(final String s) {
// other platform;
String badfs;
if ("/".equals(SEPARATOR)) {
badfs = "\\";
} else {
badfs = "/";
}
final StringBuilder ret = new StringBuilder(s.length());
for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == badfs.charAt(0)) {
ret.append(SEPARATOR);
} else {
ret.append(s.charAt(i));
}
}
return ret.toString();
}
protected Object nvl(@Nullable final Object a, final Object b) {
if (a != null) {
return a;
}
return b;
}
/**
* Format string to date path (yyyy-mm-dd/hh/(mm%10))
*
* @param dtg expected format yyyymmddhhmmss
* @return yyyy-mm-dd/hh/(mm%10)
*/
protected String datePath(@Nullable final String dtg) {
if (dtg == null) {
return TimeUtil.getDateAsPath(Instant.now());
} else {
return dtg.substring(0, 4) + "-" + // yyyy
dtg.substring(4, 6) + "-" + // mm
dtg.substring(6, 8) + "/" + // dd
dtg.substring(8, 10) + "/" + // HH
dtg.substring(10, 11) + "0"; // M0
}
}
/**
* Make a dot file name from the supplied path Ex: /path/to/file.txt --> /path/to/.file.txt
*
* @param fullName the full path and filename of the file
* @return the path as is but with a leading dot in the filename
*/
public String makeDotFile(final String fullName) {
final int fpos = fullName.lastIndexOf("/");
final int rpos = fullName.lastIndexOf("\\");
if (fpos == -1 && rpos == -1) {
return "." + fullName;
}
final int pos = Math.max(fpos, rpos);
return fullName.substring(0, pos + 1) + "." + fullName.substring(pos + 1);
}
/**
* Get the file type from the metadata or the form string passed in
*
* @param bdo IBaseDataObject
* @return the file type
*/
public static String getFileType(final IBaseDataObject bdo) {
return getAndPutFileType(bdo, null, null);
}
/**
* Get the file type from the IBaseDataObject or the form string passed in
*
* @param bdo IBaseDataObject
* @param metaData Optional map of metadata that might be modified.
* @param formsArg Optional space separated string of current forms
* @return the file type
*/
public static String getAndPutFileType(final IBaseDataObject bdo, @Nullable final Map<String, String> metaData, @Nullable final String formsArg) {
String forms = formsArg;
if (forms == null) {
forms = bdo.getStringParameter(FileTypeCheckParameter.POPPED_FORMS.getFieldName());
if (forms == null) {
forms = "";
}
}
String fileType;
if (bdo.hasParameter(FileTypeCheckParameter.FILETYPE.getFieldName())) {
fileType = bdo.getStringParameter(FileTypeCheckParameter.FILETYPE.getFieldName());
} else if (bdo.hasParameter(FileTypeCheckParameter.FINAL_ID.getFieldName())) {
fileType = bdo.getStringParameter(FileTypeCheckParameter.FINAL_ID.getFieldName());
logger.debug("FINAL_ID FileType is ({})", fileType);
if (metaData != null) {
metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
}
} else {
if (forms.contains(" ")) {
fileType = forms.substring(0, forms.indexOf(" ")).trim();
if (metaData != null) {
metaData.put(FileTypeCheckParameter.COMPLETE_FILETYPE.getFieldName(), forms);
}
} else {
fileType = forms;
}
if (StringUtils.isEmpty(fileType)) {
if (bdo.hasParameter(FileTypeCheckParameter.FONT_ENCODING.getFieldName())) {
fileType = TEXT;
} else {
fileType = UNKNOWN;
}
}
if (metaData != null) {
metaData.put(FileTypeCheckParameter.FILETYPE.getFieldName(), fileType);
}
}
if (UNKNOWN.equals(fileType) && forms.contains("MSWORD")) {
fileType = "MSWORD_FRAGMENT";
}
if ("QUOTED-PRINTABLE".equals(fileType) || fileType.startsWith(PREFIXES_LANG) || fileType.startsWith("ENCODING(")) {
fileType = TEXT;
}
return fileType;
}
/**
* Parameters that are used to determine filetype in {@link #getAndPutFileType(IBaseDataObject, Map, String)}
*/
public enum FileTypeCheckParameter {
COMPLETE_FILETYPE("COMPLETE_FILETYPE"), FILETYPE("FILETYPE"), FINAL_ID("FINAL_ID"), FONT_ENCODING("FontEncoding"), POPPED_FORMS(
"POPPED_FORMS");
final String fieldName;
FileTypeCheckParameter(String fieldName) {
this.fieldName = fieldName;
}
public String getFieldName() {
return fieldName;
}
}
/**
* Extract the ID from the payload. The ID from the payload is specified in the cfg file. An ID = SHORTNAME will use the
* shortname. An ID = AUTO_GENERATED_ID will use an auto gen uuid. If no id value is found, defaults to using an auto
* generated id.
*
* @param d the payload
* @param tld if a param is specified and cannot be found in the d method parameter, tries to get param from tld.
* @return the id based the best ID available, shortname or auto gen id. If no id is found, defaults to auto gen.
*/
public String getBestId(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
for (final String s : this.idTokens) {
if (AUTO_GENERATED_ID.equals(s)) {
String parentAutoGeneratedId = null;
if (tld != null) {
parentAutoGeneratedId = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
}
if (StringUtils.isBlank(parentAutoGeneratedId)) {
String uuid = getRandomUuid(d);
if (tld != null) {
tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
}
if (!StringUtils.isBlank(uuid)) {
final String component = d.shortName();
final int pos = component.indexOf(Family.SEP);
if (pos > -1) {
uuid += component.substring(pos);
}
return uuid;
}
}
String uuid = null;
// try getting param from the tld
if (!StringUtils.isBlank(parentAutoGeneratedId) && (tld != null)) {
uuid = tld.getStringParameter(PARENT_AUTO_GENERATED_ID);
}
if (!StringUtils.isBlank(uuid)) {
final String component = d.shortName();
final int pos = component.indexOf(Family.SEP);
if (pos > -1) {
uuid += component.substring(pos);
}
d.setParameter(AUTO_GENERATED_ID, "yes");
return uuid;
}
}
if (SHORTNAME.equals(s)) {
final String shortName = d.shortName();
// if shortname is not blank use it, if blank move on...
if (!StringUtils.isBlank(shortName)) {
return shortName;
}
String path = d.getStringParameter(s);
// try getting shortname from the tld
if (StringUtils.isBlank(path) && (tld != null)) {
path = tld.getStringParameter(s);
}
if (!StringUtils.isBlank(path)) {
final String component = d.shortName();
final int pos = component.indexOf(Family.SEP);
if (pos > -1) {
path += component.substring(pos);
}
return path;
}
}
// the param from the tld has priority over any child
if ((tld != null) && !StringUtils.isBlank(tld.getStringParameter(s))) {
String path = tld.getStringParameter(s);
if (!StringUtils.isBlank(path)) {
final String component = d.shortName();
final int pos = component.indexOf(Family.SEP);
if (pos > -1) {
path += component.substring(pos);
}
return path;
}
}
// if the param is not in the tld
if (!StringUtils.isBlank(d.getStringParameter(s))) {
return d.getStringParameter(s);
}
}
// if nothing works, auto gen an id
final String uuid = getRandomUuid(d);
if (tld != null) {
tld.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
}
return uuid;
}
/**
* Creates a UUID. Includes a prefix if specified.
*
* @param d Adds a parameter indicating an auto gen id.
* @return uuid
*/
private String getRandomUuid(final IBaseDataObject d) {
String uuid = UUID.randomUUID().toString();
// use the prefix
if (!StringUtils.isBlank(this.autoGeneratedIdPrefix)) {
uuid = uuid.substring(this.autoGeneratedIdPrefix.length());
uuid = this.autoGeneratedIdPrefix + uuid;
}
d.setParameter(PARENT_AUTO_GENERATED_ID, uuid);
d.setParameter(AUTO_GENERATED_ID, "yes");
return uuid;
}
/**
* Extract language or a default value
*
* @param d the payload
* @return the language or the default value (never null)
*/
public String getLanguage(final IBaseDataObject d) {
String lang = d.getStringParameter("LANGUAGE");
if (lang == null) {
lang = "NONE";
}
return lang;
}
/**
* Handle data
*
* @param d the data object in question
* @return the Date when the event occurred
*/
public Date getEventDate(final IBaseDataObject d, @Nullable final IBaseDataObject tld) {
Date eventDate = extractEventDateFrom(d, false);
if (eventDate == null && tld != null) {
eventDate = extractEventDateFrom(tld, this.defaultEventDateToNow);
}
return eventDate;
}
@Nullable
public Date extractEventDateFrom(final IBaseDataObject d, final boolean lastResortDefault) {
for (final String paramName : this.dateTokens) {
final String value = d.getStringParameter(paramName);
if (value != null) {
try {
ZonedDateTime zdt = FlexibleDateTimeParser.parse(value, DATE_ISO_8601);
if (zdt == null) {
logger.debug("FlexibleDateTimeParser returned null trying to parse EventDate");
} else {
return Date.from(zdt.toInstant());
}
} catch (DateTimeParseException ex) {
logger.debug("Cannot parse EventDate", ex);
}
}
}
// Default to current system time if last resort
return lastResortDefault ? Date.from(Instant.now()) : null;
}
/**
* Utilizes the static methods getFullFilepathsFromParams and getFileExtensions to extract the file extensions from all
* the filenames of the object of a given {@link IBaseDataObject}. If one or more file extensions are extracted, the
* IBaseDataObject's FILEXT parameter is set as the unique set of extracted file extensions, converted to lowercase.
*
* @param p IBaseDataObject to process
*
*/
void extractUniqueFileExtensions(IBaseDataObject p) {
List<String> filenames = getFullFilepathsFromParams(p);
Set<String> extensions = getFileExtensions(filenames, this.maxFilextLen);
if (!extensions.isEmpty()) {
p.setParameter(FILEXT, extensions);
}
}
/**
* Given a list of filenames, extract and return a set of non-blank file extensions converted to lowercase.
*
* @param filenames The list of filenames to examine
* @param maxFilextLen The maximum size we want a file extension to be
* @return A set of unique file extensions from the filename list
*/
public static Set<String> getFileExtensions(List<String> filenames, int maxFilextLen) {
final Set<String> extensions = new HashSet<>();
for (String filename : filenames) {
// add the file extension if it is smaller than maxFilextLen
final String fext = FilenameUtils.getExtension(filename);
if (StringUtils.isNotBlank(fext) && fext.length() <= maxFilextLen) {
extensions.add(fext.toLowerCase(Locale.getDefault()));
}
}
return extensions;
}
/**
* Checks the Original-Filename and FILE_ABSOLUTEPATH for the filename of the object. Returns a list with the non-empty
* strings found in these fields. If nothing is found in either field, return an empty list.
*
* @param d The IBDO
* @return The list of filenames found in the field Original-Filename or FILE_ABSOLUTEPATH
*/
public static List<String> getFullFilepathsFromParams(IBaseDataObject d) {
return getFullFilepathsFromParams(d, new String[] {ORIGINAL_FILENAME, FILE_ABSOLUTEPATH});
}
/**
* Uses the specified list of fields to check for filenames of the object. Returns a list with the non-empty strings
* found in these fields. If nothing is found in either field, return an empty list.
*
* @param d The IBDO
* @param filenameFields The list of fields on the IBDO to check
* @return The list of filenames found in the list of fields on the IBDO
* @deprecated use {@link #getFullFilepathsFromParams(IBaseDataObject, List)}
*/
@Deprecated
@SuppressWarnings("AvoidObjectArrays")
public static List<String> getFullFilepathsFromParams(IBaseDataObject d, String[] filenameFields) {
return getFullFilepathsFromParams(d, Arrays.asList(filenameFields));
}
/**
* Uses the specified list of fields to check for filenames of the object. Returns a list with the non-empty strings
* found in these fields. If nothing is found in either field, return an empty list.
*
* @param d The IBDO
* @param filenameFields The list of fields on the IBDO to check
* @return The list of filenames found in the list of fields on the IBDO
*/
public static List<String> getFullFilepathsFromParams(IBaseDataObject d, List<String> filenameFields) {
List<String> filenames = new ArrayList<>();
for (String ibdoField : filenameFields) {
if (d.hasParameter(ibdoField)) {
for (Object filename : d.getParameter(ibdoField)) {
String stringFileName = (String) filename;
if (StringUtils.isNotBlank(stringFileName)) {
filenames.add(stringFileName);
}
}
}
}
return filenames;
}
/**
* Process metadata before doing any output
*
* @param payloadList list of items being dropped off that may need initial metadata computations
*/
public void processMetadata(final List<IBaseDataObject> payloadList) {
// Keep track of parent's filetype to output; relies on the attachments being sorted
final Map<String, String> parentTypes = new HashMap<>();
final IBaseDataObject tld = payloadList.get(0);
final List<String> extendedFileTypes = new ArrayList<>();
parentTypes.put("1", tld.getFileType());
for (int i = 0; i < parentParams.size(); i++) {
final String param = parentParams.get(i);
if (tld.hasParameter(param)) {
parentTypes.put("1" + param, tld.getStringParameter(param));
}
}
for (final IBaseDataObject p : payloadList) {
final int level = StringUtils.countMatches(p.shortName(), Family.SEP) + 1;
// save specified metadata items for children to grab
parentTypes.put("" + level, p.getFileType());
extractUniqueFileExtensions(p);
if (p.getStringParameter(EXTENDED_FILETYPE) == null) {
extendedFileTypes.clear();
for (final Map.Entry<String, Collection<Object>> entry : p.getParameters().entrySet()) {
final String key = entry.getKey();
if (key != null && key.endsWith("_FILETYPE")) {
for (final Object value : entry.getValue()) {
final String vs = value.toString();
if (!extendedFileTypes.contains(vs)) {
extendedFileTypes.add(vs);
}
}
}
}
if (!extendedFileTypes.isEmpty()) {
final StringBuilder extft = new StringBuilder(getFileType(p));
for (int j = 0; j < extendedFileTypes.size(); j++) {
final String s = extendedFileTypes.get(j);
extft.append("//").append(s);
}
p.setParameter(EXTENDED_FILETYPE, extft.toString());
}
}
for (int j = 0; j < parentParams.size(); j++) {
final String param = parentParams.get(j);
if (p.hasParameter(param)) {
parentTypes.put("" + level + param, p.getStringParameter(param));
} else {
// Must clear to keep my children from getting their uncle's value
parentTypes.remove("" + level + param);
}
}
if (level > 1) {
// then it might want some PARENT info
final int parentLevel = level - 1;
final String pType = parentTypes.get("" + parentLevel);
if (StringUtils.isNotEmpty(pType)) {
p.setParameter(PARENT_FILETYPE, pType);
} else {
p.setParameter(PARENT_FILETYPE, parentTypes.get("1"));
}
for (int j = 0; j < parentParams.size(); j++) {
final String param = parentParams.get(j);
int plvl = parentLevel;
while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
plvl--;
}
if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
p.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
}
}
}
if (p.hasExtractedRecords()) {
final List<IBaseDataObject> childObjList = p.getExtractedRecords();
childObjList.sort(new ShortNameComparator());
for (final IBaseDataObject child : childObjList) {
final int parentLevel = StringUtils.countMatches(child.shortName(), Family.SEP);
final String parentFileType = parentTypes.get("" + parentLevel);
if (parentFileType != null) {
child.setParameter(PARENT_FILETYPE, parentFileType);
}
for (int k = 0; k < parentParams.size(); k++) {
final String param = parentParams.get(k);
int plvl = parentLevel;
while (plvl > 1 && !parentTypes.containsKey("" + plvl + param)) {
plvl--;
}
if (StringUtils.isNotBlank(parentTypes.get(plvl + param))) {
child.setParameter("PARENT_" + param, parentTypes.get(plvl + param));
}
}
}
}
}
}
}