View Javadoc
1   package emissary.output.filter;
2   
3   import emissary.config.Configurator;
4   import emissary.core.IBaseDataObject;
5   import emissary.core.channels.SeekableByteChannelFactory;
6   import emissary.directory.DirectoryEntry;
7   import emissary.output.io.DateFilterFilenameGenerator;
8   import emissary.util.TimeUtil;
9   
10  import com.fasterxml.jackson.annotation.JsonFilter;
11  import com.fasterxml.jackson.annotation.JsonIgnore;
12  import com.fasterxml.jackson.annotation.JsonInclude;
13  import com.fasterxml.jackson.annotation.JsonProperty;
14  import com.fasterxml.jackson.core.JsonGenerator;
15  import com.fasterxml.jackson.databind.BeanDescription;
16  import com.fasterxml.jackson.databind.JavaType;
17  import com.fasterxml.jackson.databind.JsonSerializer;
18  import com.fasterxml.jackson.databind.MapperFeature;
19  import com.fasterxml.jackson.databind.Module;
20  import com.fasterxml.jackson.databind.ObjectMapper;
21  import com.fasterxml.jackson.databind.SerializerProvider;
22  import com.fasterxml.jackson.databind.module.SimpleModule;
23  import com.fasterxml.jackson.databind.ser.BeanSerializerFactory;
24  import com.fasterxml.jackson.databind.ser.PropertyWriter;
25  import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
26  import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
27  import com.fasterxml.jackson.databind.ser.std.MapProperty;
28  import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
29  import jakarta.annotation.Nullable;
30  import org.apache.commons.collections4.CollectionUtils;
31  
32  import java.io.IOException;
33  import java.time.Instant;
34  import java.util.Collection;
35  import java.util.List;
36  import java.util.Locale;
37  import java.util.Map;
38  import java.util.Set;
39  import java.util.TreeSet;
40  import java.util.UUID;
41  
42  import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY;
43  
44  /**
45   * JSON Output filter using Jackson
46   */
47  public class JsonOutputFilter extends AbstractRollableFilter {
48  
49      protected Set<String> denylistFields = new TreeSet<>();
50      protected Set<String> denylistPrefixes = new TreeSet<>();
51      protected Set<String> allowlistFields = new TreeSet<>();
52      protected Set<String> allowlistPrefixes = new TreeSet<>();
53      protected Map<String, Set<String>> denylistValues;
54      protected Set<String> stripPrefixes;
55  
56      protected boolean emitPayload = true;
57  
58      protected ObjectMapper jsonMapper;
59  
60      @Override
61      public void initialize(final Configurator theConfigG, @Nullable final String filterName, final Configurator theFilterConfig) {
62          if (filterName == null) {
63              setFilterName("JSON");
64          }
65          super.initialize(theConfigG, filterName, theFilterConfig);
66          this.allowlistFields.addAll(this.filterConfig.findEntries("EXTRA_PARAM"));
67          this.allowlistPrefixes.addAll(this.filterConfig.findEntries("EXTRA_PREFIX"));
68          this.denylistFields.addAll(this.filterConfig.findEntries("DENYLIST_FIELD"));
69          this.denylistPrefixes.addAll(this.filterConfig.findEntries("DENYLIST_PREFIX"));
70          this.denylistValues = this.filterConfig.findStringMatchMultiMap("DENYLIST_VALUE_");
71          this.stripPrefixes = this.filterConfig.findEntriesAsSet("STRIP_PARAM_PREFIX");
72          this.emitPayload = this.filterConfig.findBooleanEntry("EMIT_PAYLOAD", true);
73          initFilenameGenerator();
74          initJsonMapper();
75      }
76  
77      @Override
78      protected void initFilenameGenerator() {
79          this.fileNameGenerator = new DateFilterFilenameGenerator("json");
80      }
81  
82      /**
83       * Initialize the Jackson json object mapper
84       */
85      protected void initJsonMapper() {
86          jsonMapper = new ObjectMapper();
87          jsonMapper.registerModule(new IbdoModule());
88          jsonMapper.registerModule(new JavaTimeModule());
89          jsonMapper.addMixIn(IBaseDataObject.class, emitPayload ? IbdoPayloadMixin.class : IbdoParameterMixin.class);
90          // the id in addFilter must match the annotation for JsonFilter
91          jsonMapper.setFilterProvider(new SimpleFilterProvider().addFilter("param_filter", new IbdoParameterFilter()));
92      }
93  
94      @Override
95      public byte[] convert(final List<IBaseDataObject> list, final Map<String, Object> params) throws IOException {
96          return jsonMapper.writeValueAsBytes(list);
97      }
98  
99      class IbdoParameterFilter extends SimpleBeanPropertyFilter {
100 
101         protected final boolean outputAll;
102         protected final boolean emptyDenylist;
103         protected final boolean denylistStar;
104         protected final boolean emptyAllowlist;
105         protected final boolean allowlistStar;
106         private static final char KEY_REPLACEMENT = '_';
107 
108         public IbdoParameterFilter() {
109             // if all collections are empty, then output everything
110             this.allowlistStar = (allowlistFields.contains("*") || allowlistFields.contains("ALL"));
111             this.denylistStar = (denylistFields.contains("*") || denylistFields.contains("ALL"));
112             this.emptyDenylist = CollectionUtils.isEmpty(denylistFields) && CollectionUtils.isEmpty(denylistPrefixes);
113             this.emptyAllowlist = CollectionUtils.isEmpty(allowlistFields) && CollectionUtils.isEmpty(allowlistPrefixes);
114             this.outputAll = emptyDenylist && (allowlistStar || emptyAllowlist);
115         }
116 
117         @Override
118         public void serializeAsField(Object pojo, JsonGenerator jgen, SerializerProvider provider, PropertyWriter writer) throws Exception {
119 
120             String key = writer.getName();
121             @SuppressWarnings("unchecked")
122             Collection<Object> values = (Collection<Object>) ((Map<?, ?>) pojo).get(key);
123 
124             if (includeParameter(key)) {
125                 Collection<Object> write = filter(key, values);
126                 if (CollectionUtils.isNotEmpty(write)) {
127                     // customize the key
128                     jgen.writeFieldName(transform(key));
129 
130                     // only write the element
131                     ((MapProperty) writer).setValue(write);
132                     writer.serializeAsElement(write, jgen, provider);
133                 }
134             }
135         }
136 
137         protected boolean includeParameter(String key) {
138             if (outputAll) {
139                 return true;
140             }
141 
142             // check the allow/deny list first
143             if (denylistFields.contains(key)) {
144                 return false;
145             } else if (allowlistFields.contains(key)) {
146                 return true;
147             }
148 
149             // see if there is a hit on the denylist prefix
150             for (final String prefix : denylistPrefixes) {
151                 if (key.startsWith(prefix)) {
152                     return false;
153                 }
154             }
155 
156             // omit/emit all parameters if '*' or 'ALL'
157             if (denylistStar) {
158                 return false;
159             } else if (allowlistStar) {
160                 return true;
161             }
162 
163             // there is a hit on the allow-list prefix, but it is on the deny-list
164             for (final String prefix : allowlistPrefixes) {
165                 if (key.startsWith(prefix)) {
166                     return true;
167                 }
168             }
169 
170             // if we were only given an deny-list, output all keys
171             return emptyAllowlist;
172         }
173 
174         protected Collection<Object> filter(String key, Collection<Object> values) {
175             Set<Object> keep = new TreeSet<>();
176             for (final Object value : values) {
177                 if (!(denylistValues.containsKey(key) && denylistValues.get(key).contains(value.toString()))) {
178                     keep.add(value);
179                 }
180             }
181             return keep;
182         }
183 
184         protected String transform(String name) {
185             return normalize(strip(name.toUpperCase(Locale.getDefault())));
186         }
187 
188         protected String strip(String name) {
189             for (final String prefix : stripPrefixes) {
190                 if (name.startsWith(prefix)) {
191                     return name.substring(prefix.length());
192                 }
193             }
194             return name;
195         }
196 
197         protected String normalize(String name) {
198             boolean changed = false;
199             char[] ch = name.toCharArray();
200             for (int i = 0; i < ch.length; i++) {
201                 if (!Character.isLetterOrDigit(ch[i]) && Character.compare(ch[i], '_') != 0 && Character.compare(ch[i], '.') != 0) {
202                     ch[i] = KEY_REPLACEMENT;
203                     changed = true;
204                 }
205             }
206             if (changed) {
207                 return new String(ch);
208             }
209 
210             return name;
211         }
212     }
213 
214     /**
215      * Ibdo {@link Module} implementation that allows registration of serializers
216      */
217     class IbdoModule extends SimpleModule {
218         private static final long serialVersionUID = -8129967131240053241L;
219 
220         public IbdoModule() {
221             addSerializer(IBaseDataObject.class, new IbdoSerializer());
222         }
223     }
224 
225     /**
226      * Add some fields to the ibdo before output This is only needed if custom fields need to be written for the ibdo
227      */
228     class IbdoSerializer extends JsonSerializer<IBaseDataObject> {
229 
230         @Override
231         public void serialize(IBaseDataObject ibdo, JsonGenerator jgen, SerializerProvider provider) throws IOException {
232             jgen.writeStartObject();
233             JavaType javaType = provider.constructType(IBaseDataObject.class);
234             BeanDescription beanDesc = provider.getConfig().introspect(javaType);
235             JsonSerializer<Object> serializer = BeanSerializerFactory.instance.findBeanOrAddOnSerializer(provider, javaType, beanDesc,
236                     provider.isEnabled(MapperFeature.USE_STATIC_TYPING));
237 
238             // add some custom fields here
239             jgen.writeObjectField("id", dropOffUtil.getBestIdFrom(ibdo));
240             jgen.writeObjectField("processedTimestamp", TimeUtil.getCurrentDateFullISO8601());
241 
242             serializer.unwrappingSerializer(null).serialize(ibdo, jgen, provider);
243             jgen.writeEndObject();
244         }
245     }
246 
247     /**
248      * This class is used so we do not have to annotate the IBaseDataObject. Set custom annotations on the method signatures
249      * to include/exclude fields in the ibdo.
250      */
251     abstract static class IbdoMixin {
252         @JsonProperty("internalId")
253         abstract UUID getInternalId();
254 
255         @JsonProperty("creationTimestamp")
256         abstract Instant getCreationTimestamp();
257 
258         @JsonProperty("shortName")
259         abstract String shortName();
260 
261         @JsonProperty("parameters")
262         @JsonFilter("param_filter")
263         abstract Map<String, Collection<Object>> getParameters();
264 
265         @JsonProperty("members")
266         @JsonInclude(NON_EMPTY)
267         abstract List<IBaseDataObject> getExtractedRecords();
268 
269         @JsonIgnore
270         abstract SeekableByteChannelFactory getChannelFactory();
271 
272         @JsonIgnore
273         abstract int dataLength();
274 
275         @JsonIgnore
276         abstract String getHeaderEncoding();
277 
278         @JsonIgnore
279         abstract int getNumChildren();
280 
281         @JsonIgnore
282         abstract int getNumSiblings();
283 
284         @JsonIgnore
285         abstract int getBirthOrder();
286 
287         @JsonIgnore
288         abstract String getFontEncoding();
289 
290         @JsonIgnore
291         abstract Map<String, String> getCookedParameters();
292 
293         @JsonIgnore
294         abstract Set<String> getParameterKeys();
295 
296         @JsonIgnore
297         abstract boolean isFileTypeEmpty();
298 
299         @JsonIgnore
300         abstract String getFileType();
301 
302         @JsonIgnore
303         abstract int getNumAlternateViews();
304 
305         @JsonIgnore
306         abstract Set<String> getAlternateViewNames();
307 
308         @JsonIgnore
309         abstract boolean isBroken();
310 
311         @JsonIgnore
312         abstract String getFilename();
313 
314         @JsonIgnore
315         abstract List<String> getAllCurrentForms();
316 
317         @JsonIgnore
318         abstract DirectoryEntry getLastPlaceVisited();
319 
320         @JsonIgnore
321         abstract DirectoryEntry getPenultimatePlaceVisited();
322 
323         @JsonIgnore
324         abstract int getPriority();
325 
326         @JsonIgnore
327         abstract int getExtractedRecordCount();
328 
329         @JsonIgnore
330         abstract boolean isOutputable();
331 
332         @JsonIgnore
333         abstract String getBroken();
334 
335         @JsonIgnore
336         abstract String getProcessingError();
337     }
338 
339     abstract static class IbdoParameterMixin extends IbdoMixin {
340         @JsonIgnore
341         abstract byte[] data();
342 
343         @JsonIgnore
344         abstract Map<String, byte[]> getAlternateViews();
345     }
346 
347     abstract static class IbdoPayloadMixin extends IbdoMixin {
348         @JsonProperty("payload")
349         @JsonInclude(NON_EMPTY)
350         abstract byte[] data();
351 
352         @JsonProperty("views")
353         @JsonInclude(NON_EMPTY)
354         abstract Map<String, byte[]> getAlternateViews();
355     }
356 }