Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019
020package org.apache.oozie.util;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Locale;
028import java.util.Map;
029import java.util.regex.Matcher;
030import java.util.regex.Pattern;
031
032import org.apache.commons.lang.StringUtils;
033import org.apache.oozie.service.ConfigurationService;
034import org.apache.oozie.util.LogLine.MATCHED_PATTERN;
035
036import com.google.common.annotations.VisibleForTesting;
037
038/**
039 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if
040 * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|")
041 * jobId appName actionId token
042 */
043public class XLogFilter {
044
045    private static final int LOG_TIME_BUFFER = 2; // in min
046    public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration";
047    public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration";
048    private Map<String, Integer> logLevels;
049    private final Map<String, String> filterParams;
050    private static List<String> parameters = new ArrayList<String>();
051    private boolean noFilter;
052    private Pattern filterPattern;
053    private XLogUserFilterParam userLogFilter;
054    private Date endDate;
055    private Date startDate;
056    private boolean isActionList = false;
057    private String formattedEndDate;
058    private String formattedStartDate;
059
060    // TODO Patterns to be read from config file
061    private static final String DEFAULT_REGEX = "[^\\]]*";
062
063    public static final String ALLOW_ALL_REGEX = "(.*)";
064    private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
065    private static final String WHITE_SPACE_REGEX = "\\s+";
066    private static final String LOG_LEVEL_REGEX = "(\\w+)";
067    static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
068            + WHITE_SPACE_REGEX;
069    private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
070
071    public XLogFilter() {
072        this(new XLogUserFilterParam());
073    }
074
075    public XLogFilter(XLogUserFilterParam userLogFilter) {
076        filterParams = new HashMap<String, String>();
077        for (int i = 0; i < parameters.size(); i++) {
078            filterParams.put(parameters.get(i), DEFAULT_REGEX);
079        }
080        logLevels = null;
081        noFilter = true;
082        filterPattern = null;
083        setUserLogFilter(userLogFilter);
084    }
085
086    public void setLogLevel(String logLevel) {
087        if (logLevel != null && logLevel.trim().length() > 0) {
088            this.logLevels = new HashMap<String, Integer>();
089            String[] levels = logLevel.split("\\|");
090            for (int i = 0; i < levels.length; i++) {
091                String s = levels[i].trim().toUpperCase();
092                try {
093                    XLog.Level.valueOf(s);
094                }
095                catch (Exception ex) {
096                    continue;
097                }
098                this.logLevels.put(levels[i].toUpperCase(), 1);
099            }
100        }
101    }
102
103    public void setParameter(String filterParam, String value) {
104        if (filterParams.containsKey(filterParam)) {
105            noFilter = false;
106            filterParams.put(filterParam, value);
107        }
108    }
109
110    public static void defineParameter(String filterParam) {
111        parameters.add(filterParam);
112    }
113
114    public boolean isFilterPresent() {
115        if (noFilter && logLevels == null) {
116            return false;
117        }
118        return true;
119    }
120
121    /**
122     * Checks if the logLevel and logMessage goes through the logFilter.
123     *
124     * @param logParts
125     * @return
126     */
127    public boolean splitsMatches(LogLine logLine) {
128        // Check whether logLine matched with filter
129        if (logLine.getMatchedPattern() != MATCHED_PATTERN.SPLIT) {
130            return false;
131        }
132        ArrayList<String> logParts = logLine.getLogParts();
133        if (getStartDate() != null) {
134            if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
135                return false;
136            }
137        }
138        String logLevel = logParts.get(1);
139        if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase(Locale.ENGLISH))) {
140            // line contains the permitted logLevel
141            return true;
142        }
143        else {
144            return false;
145        }
146    }
147
148    /**
149     * Checks if the logLevel and logMessage goes through the logFilter.
150     *
151     * @param logParts
152     * @return
153     */
154    public boolean matches(ArrayList<String> logParts) {
155        if (getStartDate() != null) {
156            if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
157                return false;
158            }
159        }
160        String logLevel = logParts.get(1);
161        String logMessage = logParts.get(2);
162        if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
163            Matcher logMatcher = filterPattern.matcher(logMessage);
164            return logMatcher.matches();
165        }
166        else {
167            return false;
168        }
169    }
170
171    /**
172     * Splits the log line into timestamp, logLevel and remaining log message.
173     * Returns array containing timestamp, logLevel, and logMessage if the
174     * pattern matches i.e A new log statement, else returns null.
175     *
176     * @param logLine
177     * @return Array containing log level and log message
178     */
179    public ArrayList<String> splitLogMessage(String logLine) {
180        Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
181        if (splitter.matches()) {
182            ArrayList<String> logParts = new ArrayList<String>();
183            logParts.add(splitter.group(1));// timestamp
184            logParts.add(splitter.group(2));// log level
185            logParts.add(splitter.group(3));// Log Message
186            return logParts;
187        }
188        else {
189            return null;
190        }
191    }
192
193    /**
194     * If <code>logLine</code> matches with <code>splitPattern</code>,
195     * <ol>
196     * <li>Split the log line into timestamp, logLevel and remaining log
197     * message.</li>
198     * <li>Record the parts of message in <code>logLine</code> to avoid regex
199     * matching in future.</li>
200     * <li>Record the pattern to which <code>logLine</code> has matched.</li>
201     * </ol>
202     * @param logLine
203     * @param splitPattern
204     */
205    public void splitLogMessage(LogLine logLine, Pattern splitPattern) {
206        Matcher splitterWithJobId = splitPattern.matcher(logLine.getLine());
207        Matcher allowAll = SPLITTER_PATTERN.matcher(logLine.getLine());
208        if (splitterWithJobId.matches()) {
209            ArrayList<String> logParts = new ArrayList<String>(3);
210            logParts.add(splitterWithJobId.group(1));// timestamp
211            logParts.add(splitterWithJobId.group(2));// log level
212            logParts.add(splitterWithJobId.group(3));// log message
213            logLine.setLogParts(logParts);
214            logLine.setMatchedPattern(MATCHED_PATTERN.SPLIT);
215        }
216        else if (allowAll.matches()) {
217            logLine.setMatchedPattern(MATCHED_PATTERN.GENENRIC);
218        }
219        else {
220            logLine.setMatchedPattern(MATCHED_PATTERN.NONE);
221        }
222    }
223
224    /**
225     * Constructs the regular expression according to the filter and assigns it
226     * to fileterPattarn. ".*" will be assigned if no filters are set.
227     */
228    public void constructPattern() {
229        if (noFilter && logLevels == null) {
230            filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
231            return;
232        }
233        StringBuilder sb = new StringBuilder();
234        if (noFilter) {
235            sb.append("(.*)");
236        }
237        else {
238            sb.append("(.* ");
239            for (int i = 0; i < parameters.size(); i++) {
240                sb.append(parameters.get(i) + "\\[");
241                sb.append(filterParams.get(parameters.get(i)) + "\\] ");
242            }
243            sb.append(".*)");
244        }
245        if (!StringUtils.isEmpty(userLogFilter.getSearchText())) {
246            sb.append(userLogFilter.getSearchText() + ".*");
247        }
248        filterPattern = Pattern.compile(sb.toString());
249    }
250
251    public static void reset() {
252        parameters.clear();
253    }
254
255    public final Map<String, String> getFilterParams() {
256        return filterParams;
257    }
258
259    public XLogUserFilterParam getUserLogFilter() {
260        return userLogFilter;
261    }
262
263    public void setUserLogFilter(XLogUserFilterParam userLogFilter) {
264        this.userLogFilter = userLogFilter;
265        setLogLevel(userLogFilter.getLogLevel());
266    }
267
268    public Date getEndDate() {
269        return endDate;
270    }
271
272    public String getFormattedEndDate() {
273        return formattedEndDate;
274    }
275
276    public String getFormattedStartDate() {
277        return formattedStartDate;
278    }
279
280    public Date getStartDate() {
281        return startDate;
282    }
283
284    public boolean isDebugMode() {
285        return userLogFilter.isDebug();
286    }
287
288    public int getLogLimit() {
289        return userLogFilter.getLimit();
290    }
291
292    public String getDebugMessage() {
293        return "Log start time = " + getStartDate() + ". Log end time = " + getEndDate() + ". User Log Filter = "
294                + getUserLogFilter() + System.getProperty("line.separator");
295    }
296
297    public boolean isActionList() {
298        return isActionList;
299    }
300
301    public void setActionList(boolean isActionList) {
302        this.isActionList = isActionList;
303    }
304
305    private void calculateScanDate(Date jobStartTime, Date jobEndTime) throws IOException {
306
307        if (userLogFilter.getStartDate() != null) {
308            startDate = userLogFilter.getStartDate();
309        }
310        else if (userLogFilter.getStartOffset() != -1) {
311            startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset());
312        }
313        else {
314            startDate = jobStartTime;
315        }
316
317        if (userLogFilter.getEndDate() != null) {
318            endDate = userLogFilter.getEndDate();
319        }
320        else if (userLogFilter.getEndOffset() != -1) {
321            // If user has specified startdate as absolute then end offset will be on user start date,
322            // else end offset will be calculated on job startdate.
323            if (userLogFilter.getStartDate() != null) {
324                endDate = adjustOffset(startDate, userLogFilter.getEndOffset());
325            }
326            else {
327                endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset());
328            }
329        }
330        else {
331            endDate = jobEndTime;
332        }
333        // if recent offset is specified then start time = endtime - offset
334        if (getUserLogFilter().getRecent() != -1) {
335            startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1);
336        }
337
338        //add buffer iff dates are not asbsolute
339        if (userLogFilter.getStartDate() == null) {
340            startDate = adjustOffset(startDate, -LOG_TIME_BUFFER);
341        }
342        if (userLogFilter.getEndDate() == null) {
343            endDate = adjustOffset(endDate, LOG_TIME_BUFFER);
344        }
345
346        formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate());
347        formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate());
348    }
349
350    /**
351     * Calculate and validate date range.
352     *
353     * @param jobStartTime the job start time
354     * @param jobEndTime the job end time
355     * @throws IOException Signals that an I/O exception has occurred.
356     */
357    public void calculateAndValidateDateRange(Date jobStartTime, Date jobEndTime) throws IOException {
358        // for testcase, otherwise jobStartTime and jobEndTime will be always set
359        if (jobStartTime == null || jobEndTime == null) {
360            return;
361        }
362        calculateScanDate(jobStartTime, jobEndTime);
363
364        if (startDate.after(endDate)) {
365            throw new IOException("Start time should be less than end time. startTime = " + startDate + " endtime = "
366                    + endDate);
367        }
368        long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000);
369        if (isActionList) {
370            int actionLogDuration = ConfigurationService.getInt(MAX_ACTIONLIST_SCAN_DURATION);
371            if (actionLogDuration == -1) {
372                return;
373            }
374            if (diffHours > actionLogDuration) {
375                throw new IOException(
376                        "Request log streaming time range with action list is higher than configured. Please reduce the scan "
377                                + "time range. Input range (hours) = " + diffHours
378                                + " system allowed (hours) with action list = " + actionLogDuration);
379            }
380        }
381        else {
382            int logDuration = ConfigurationService.getInt(MAX_SCAN_DURATION);
383            if (logDuration == -1) {
384                return;
385            }
386            if (diffHours > logDuration) {
387                throw new IOException(
388                        "Request log streaming time range is higher than configured. Please reduce the scan time range. For coord"
389                                + " jobs you can provide action list to reduce log scan time range. Input range (hours) = "
390                                + diffHours + " system allowed (hours) = " + logDuration);
391            }
392        }
393    }
394
395    /**
396     * Adjust offset, offset will always be in min.
397     *
398     * @param date the date
399     * @param offset the offset
400     * @return the date
401     * @throws IOException Signals that an I/O exception has occurred.
402     */
403    public Date adjustOffset(Date date, int offset) throws IOException {
404        return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
405    }
406
407    public void setFilterPattern(Pattern filterPattern) {
408        this.filterPattern = filterPattern;
409    }
410
411    public Pattern getFilterPattern() {
412        return this.filterPattern;
413    }
414
415}