diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json new file mode 100644 index 000000000..eac5e8470 --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json @@ -0,0 +1,34 @@ +{ + "id" : "91480bc4-2a6d-49de-8f0f-5e654b10e90f", + "name" : "asa", + "parsers" : [ { + "id" : "fa555d30-0689-11f0-9048-9faa4f53dab2", + "name" : "initial_grok", + "type" : "com.cloudera.parserchains.parsers.GrokTableParser", + "config" : { + "grokPatternPath" : [ { + "grokPatternPath" : "grok/cisco_asa" + } ], + "initialGrokExpression" : [ { + "initialGrokExpression" : "%{CISCO_TAGGED_SYSLOG}" + } ], + "keyFieldName" : [ { + "keyFieldName" : "cisco_tag" + } ], + "messageFieldName" : [ { + "messageFieldName" : "message" + } ] + } + }, { + "id" : "fec11030-0693-11f0-8f14-4f461159ed6f", + "name" : "convert_timestamp", + "type" : "com.cloudera.parserchains.parsers.TimestampFormatParser", + "config" : { + "fields" : [ { + "field" : "timestamp", + "tz" : "UTC", + "format" : "MMM dd yyyy HH:mm:ss" + } ] + } + } ] +} \ No newline at end of file diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/grok/cisco_asa b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/grok/cisco_asa new file mode 100644 index 000000000..e0b809dbd --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/grok/cisco_asa @@ -0,0 +1,223 @@ +# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port:int})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +GREEDYDATA .* + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid:int}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility:int}.%{NONNEGINT:priority:int}> +SYSLOGPRIORITY <%{POSINT:syslog_pri:int}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: + +MESSAGESLOG %{SYSLOGBASE} %{DATA} + +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} + +# Log Levels +LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + +#== Cisco ASA == +CISCO_TAGGED_SYSLOG (^<%{POSINT:syslog_pri:int}>)?%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:syslog_host})?( %{SYSLOGPROG:syslog_prog})? ?:? %%{CISCOTAG:cisco_tag}%{GREEDYDATA:message} +CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} +CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) + +# Common Particles +CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted +CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* +CISCO_DIRECTION Inbound|inbound|Outbound|outbound +CISCO_INTERVAL first hit|%{INT}-second interval +CISCO_XLATE_TYPE static|dynamic +CISCO_XLATE_CREATE_TYPE outbound static|identity|portmap|regular + +# ASA-2-106001 +CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-2-106006, ASA-2-106007, ASA-2-106010 +CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port:int}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port:int}(\(%{DATA:dst_fwuser}\))? (?:on interface %{GREEDYDATA:interface}|due to %{CISCO_REASON:reason}) +# ASA-3-106014 +CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type:int}, code %{INT:icmp_code:int}\) +# ASA-6-106015 +CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-1-106021 +CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} +# ASA-4-106023 +CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-5-106100 +CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port:int}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port:int}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count:int} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-6-110002 +CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-302010 +CISCOFW302010 : %{INT:connection_count:int} in use, %{INT:connection_count_max:int} most used +# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 +CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id:int} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port:int}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port:int}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes:int})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? +# ASA-6-302020, ASA-6-302021 +CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? +# ASA-6-305011 +CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} +# ASA-3-313001, ASA-3-313004, ASA-3-313008 +CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type:int}, code=%{INT:icmp_code:int} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? +# ASA-4-313005 +CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type:int}, code %{INT:err_icmp_code:int}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port:int}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port:int}(\(%{DATA:orig_dst_fwuser}\))? +# ASA-4-402117 +CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} +# ASA-4-402119 +CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking +# ASA-4-419001 +CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}, reason: %{GREEDYDATA:reason} +# ASA-4-419002 +CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} with different initial sequence number +# ASA-4-500004 +CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-602303, ASA-6-602304 +CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} +# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 +CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-713172 +CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device +# ASA-4-733100 +CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst:int} per second, max configured rate is %{INT:drop_rate_max_burst:int}; Current average rate is %{INT:drop_rate_current_avg:int} per second, max configured rate is %{INT:drop_rate_max_avg:int}; Cumulative total count is %{INT:drop_total_count:int} + + +# ASA-6-305012 +CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} +# ASA-7-609001 +CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? +# ASA-7-609002 +CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} + +# ASA-5-713041 +CISCOFW713041 : (Group = %{GREEDYDATA:group}, )?IP = %{IP:src_ip}, IKE Initiator: Rekeying Phase %{INT}, Intf %{DATA:src_interface}, IKE Peer %{DATA}\s+local Proxy Address %{IP}, remote Proxy Address %{IP},\s+Crypto map \(%{DATA}\) + +ASA_2_106001 %{CISCOFW106001:details} +ASA_2_106006 %{CISCOFW106006_106007_106010:details} +ASA_2_106007 %{CISCOFW106006_106007_106010:details} +ASA_2_106010 %{CISCOFW106006_106007_106010} +ASA_3_106014 %{CISCOFW106014} +ASA_6_106015 %{CISCOFW106015} +ASA_1_106021 %{CISCOFW106021} +ASA_4_106023 %{CISCOFW106023} +ASA_5_106100 %{CISCOFW106100} +ASA_6_110002 %{CISCOFW110002} +ASA_6_302010 %{CISCOFW302010} +ASA_6_302013 %{CISCOFW302013_302014_302015_302016} +ASA_6_302014 %{CISCOFW302013_302014_302015_302016} +ASA_6_302015 %{CISCOFW302013_302014_302015_302016} +ASA_6_302016 %{CISCOFW302013_302014_302015_302016} +ASA_6_302020 %{CISCOFW302020_302021} +ASA_6_302021 %{CISCOFW302020_302021} +ASA_6_305011 %{CISCOFW305011} +ASA_3_313001 %{CISCOFW313001_313004_313008} +ASA_3_313004 %{CISCOFW313001_313004_313008} +ASA_3_313008 %{CISCOFW313001_313004_313008} +ASA_4_313005 %{CISCOFW313005} +ASA_4_402117 %{CISCOFW402117} +ASA_4_402119 %{CISCOFW402119} +ASA_4_419001 %{CISCOFW419001} +ASA_4_419002 %{CISCOFW419002} +ASA_4_500004 %{CISCOFW500004} +ASA_6_602303 %{CISCOFW602303_602304} +ASA_6_602304 %{CISCOFW602303_602304} +ASA_7_710001 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710002 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710003 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710005 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710006 %{CISCOFW710001_710002_710003_710005_710006} +ASA_6_713172 %{CISCOFW713172} +ASA_4_733100 %{CISCOFW733100} +ASA_6_305012 %{CISCOFW305012} +ASA_7_609001 %{CISCOFW609001} +ASA_7_609002 %{CISCOFW609002} +ASA_5_713041 %{CISCOFW713041} +ASA_3_305006 : %{CISCO_XLATE_CREATE_TYPE} translation creation failed for %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{DATA:err_src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{DATA:err_dst_fwuser}\))? + +#== End Cisco ASA == \ No newline at end of file diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/samples/cisco_asa.txt b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/samples/cisco_asa.txt new file mode 100644 index 000000000..8e69dbd05 --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/basic/parse/samples/cisco_asa.txt @@ -0,0 +1,50 @@ +<166>Aug 05 2016 01:01:00 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.199/ssh for user 'root' +<166>Aug 05 2016 01:01:10 ASA : %ASA-6-302013: Built outbound TCP connection 82133946 for outside:203.0.113.70/443 to inside:192.168.1.49/53235 +<166>Aug 05 2016 01:01:20 ASA : %ASA-6-302013: Built outbound TCP connection 28488738 for outside:203.0.113.74/8080 to inside:192.168.1.139/15963 (session lasted 2509 seconds, 3 GB transferred) +<166>Aug 05 2016 01:01:30 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.48 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:01:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.35/27281 dst outside:203.0.113.86/10679 +<166>Aug 05 2016 01:01:50 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.185/33425 dst inside:203.0.113.78/43222 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:00 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.245/ssh for user 'user1' +<166>Aug 05 2016 01:02:10 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.207/6733 dst inside:203.0.113.136/24270 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:20 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.124 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:02:30 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.77 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:02:40 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.105/47867 dst inside:203.0.113.72/4011 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:50 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.237 dst inside:203.0.113.212 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:03:00 ASA : %ASA-6-302013: Built outbound TCP connection 71778197 for outside:203.0.113.60/8080 to inside:192.168.1.243/62767 (session lasted 1591 seconds, 10 GB transferred) +<166>Aug 05 2016 01:03:10 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.204 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:03:20 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.28 dst inside:203.0.113.118 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:03:30 ASA : %ASA-6-302013: Built outbound TCP connection 21235826 for outside:203.0.113.232/443 to inside:192.168.1.35/57987 +<166>Aug 05 2016 01:03:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.146/31215 dst outside:203.0.113.214/17544 +<166>Aug 05 2016 01:03:50 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.36/ssh for user 'root' +<166>Aug 05 2016 01:04:00 ASA : %ASA-6-302013: Built outbound TCP connection 65554300 for outside:203.0.113.235/443 to inside:192.168.1.153/35324 +<166>Aug 05 2016 01:04:10 ASA : %ASA-6-302013: Built outbound TCP connection 86288205 for outside:203.0.113.64/8080 to inside:192.168.1.225/24361 (session lasted 5995 seconds, 2 GB transferred) +<166>Aug 05 2016 01:04:20 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.19 dst inside:203.0.113.234 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:04:30 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.63/ssh for user 'user1' +<166>Aug 05 2016 01:04:40 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.186/ssh for user 'user1' +<166>Aug 05 2016 01:04:50 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.244/ssh for user 'user1' +<166>Aug 05 2016 01:05:00 ASA : %ASA-6-302013: Built outbound TCP connection 39234062 for outside:203.0.113.11/8080 to inside:192.168.1.89/20664 (session lasted 465 seconds, 6 GB transferred) +<166>Aug 05 2016 01:05:10 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.5/ssh for user 'admin' +<166>Aug 05 2016 01:05:20 ASA : %ASA-6-302013: Built outbound TCP connection 35055656 for outside:203.0.113.231/443 to inside:192.168.1.73/31031 +<166>Aug 05 2016 01:05:30 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.13/21008 dst inside:203.0.113.82/36378 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:05:40 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.52/ssh for user 'user1' +<166>Aug 05 2016 01:05:50 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.114 dst inside:203.0.113.249 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:06:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.254/35812 dst inside:203.0.113.4/48884 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:06:10 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.94 dst inside:203.0.113.173 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:06:20 ASA : %ASA-6-302013: Built outbound TCP connection 82554922 for outside:203.0.113.75/443 to inside:192.168.1.201/44338 +<166>Aug 05 2016 01:06:30 ASA : %ASA-6-302013: Built outbound TCP connection 42701734 for outside:203.0.113.222/443 to inside:192.168.1.205/21435 +<166>Aug 05 2016 01:06:40 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.195/40493 dst inside:203.0.113.205/62028 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:06:50 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.79/45112 dst inside:203.0.113.63/62563 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.35/21904 dst inside:203.0.113.219/50940 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:10 ASA : %ASA-6-302013: Built outbound TCP connection 49890331 for outside:203.0.113.10/8080 to inside:192.168.1.87/51767 (session lasted 3877 seconds, 6 GB transferred) +<166>Aug 05 2016 01:07:20 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.208 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:07:30 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.6/25226 dst inside:203.0.113.49/15638 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:40 ASA : %ASA-6-302013: Built outbound TCP connection 42193182 for outside:203.0.113.197/443 to inside:192.168.1.246/58966 +<166>Aug 05 2016 01:07:50 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.234 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:08:00 ASA : %ASA-6-302013: Built outbound TCP connection 49955429 for outside:203.0.113.21/443 to inside:192.168.1.96/58844 +<166>Aug 05 2016 01:08:10 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.194/30591 dst inside:203.0.113.226/19073 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:08:20 ASA : %ASA-6-302013: Built outbound TCP connection 89392405 for outside:203.0.113.165/8080 to inside:192.168.1.130/37276 (session lasted 568 seconds, 6 GB transferred) +<166>Aug 05 2016 01:08:30 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.195 dst inside:203.0.113.155 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:08:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.142/8061 dst outside:203.0.113.28/25227 +<166>Aug 05 2016 01:08:50 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.146 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:09:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.191/59896 dst inside:203.0.113.241/51735 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:09:10 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.12 dst inside:203.0.113.232 (type 3, code 3) on outside interface. diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json new file mode 100644 index 000000000..eac5e8470 --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/chains/91480bc4-2a6d-49de-8f0f-5e654b10e90f.json @@ -0,0 +1,34 @@ +{ + "id" : "91480bc4-2a6d-49de-8f0f-5e654b10e90f", + "name" : "asa", + "parsers" : [ { + "id" : "fa555d30-0689-11f0-9048-9faa4f53dab2", + "name" : "initial_grok", + "type" : "com.cloudera.parserchains.parsers.GrokTableParser", + "config" : { + "grokPatternPath" : [ { + "grokPatternPath" : "grok/cisco_asa" + } ], + "initialGrokExpression" : [ { + "initialGrokExpression" : "%{CISCO_TAGGED_SYSLOG}" + } ], + "keyFieldName" : [ { + "keyFieldName" : "cisco_tag" + } ], + "messageFieldName" : [ { + "messageFieldName" : "message" + } ] + } + }, { + "id" : "fec11030-0693-11f0-8f14-4f461159ed6f", + "name" : "convert_timestamp", + "type" : "com.cloudera.parserchains.parsers.TimestampFormatParser", + "config" : { + "fields" : [ { + "field" : "timestamp", + "tz" : "UTC", + "format" : "MMM dd yyyy HH:mm:ss" + } ] + } + } ] +} \ No newline at end of file diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/grok/cisco_asa b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/grok/cisco_asa new file mode 100644 index 000000000..e0b809dbd --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/grok/cisco_asa @@ -0,0 +1,223 @@ +# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port:int})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +GREEDYDATA .* + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid:int}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility:int}.%{NONNEGINT:priority:int}> +SYSLOGPRIORITY <%{POSINT:syslog_pri:int}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: + +MESSAGESLOG %{SYSLOGBASE} %{DATA} + +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} + +# Log Levels +LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + +#== Cisco ASA == +CISCO_TAGGED_SYSLOG (^<%{POSINT:syslog_pri:int}>)?%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:syslog_host})?( %{SYSLOGPROG:syslog_prog})? ?:? %%{CISCOTAG:cisco_tag}%{GREEDYDATA:message} +CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} +CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) + +# Common Particles +CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted +CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* +CISCO_DIRECTION Inbound|inbound|Outbound|outbound +CISCO_INTERVAL first hit|%{INT}-second interval +CISCO_XLATE_TYPE static|dynamic +CISCO_XLATE_CREATE_TYPE outbound static|identity|portmap|regular + +# ASA-2-106001 +CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-2-106006, ASA-2-106007, ASA-2-106010 +CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port:int}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port:int}(\(%{DATA:dst_fwuser}\))? (?:on interface %{GREEDYDATA:interface}|due to %{CISCO_REASON:reason}) +# ASA-3-106014 +CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type:int}, code %{INT:icmp_code:int}\) +# ASA-6-106015 +CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-1-106021 +CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} +# ASA-4-106023 +CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-5-106100 +CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port:int}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port:int}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count:int} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-6-110002 +CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-302010 +CISCOFW302010 : %{INT:connection_count:int} in use, %{INT:connection_count_max:int} most used +# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 +CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id:int} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port:int}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port:int}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes:int})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? +# ASA-6-302020, ASA-6-302021 +CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? +# ASA-6-305011 +CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} +# ASA-3-313001, ASA-3-313004, ASA-3-313008 +CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type:int}, code=%{INT:icmp_code:int} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? +# ASA-4-313005 +CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type:int}, code %{INT:err_icmp_code:int}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port:int}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port:int}(\(%{DATA:orig_dst_fwuser}\))? +# ASA-4-402117 +CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} +# ASA-4-402119 +CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking +# ASA-4-419001 +CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}, reason: %{GREEDYDATA:reason} +# ASA-4-419002 +CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} with different initial sequence number +# ASA-4-500004 +CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-602303, ASA-6-602304 +CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} +# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 +CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-713172 +CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device +# ASA-4-733100 +CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst:int} per second, max configured rate is %{INT:drop_rate_max_burst:int}; Current average rate is %{INT:drop_rate_current_avg:int} per second, max configured rate is %{INT:drop_rate_max_avg:int}; Cumulative total count is %{INT:drop_total_count:int} + + +# ASA-6-305012 +CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} +# ASA-7-609001 +CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? +# ASA-7-609002 +CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} + +# ASA-5-713041 +CISCOFW713041 : (Group = %{GREEDYDATA:group}, )?IP = %{IP:src_ip}, IKE Initiator: Rekeying Phase %{INT}, Intf %{DATA:src_interface}, IKE Peer %{DATA}\s+local Proxy Address %{IP}, remote Proxy Address %{IP},\s+Crypto map \(%{DATA}\) + +ASA_2_106001 %{CISCOFW106001:details} +ASA_2_106006 %{CISCOFW106006_106007_106010:details} +ASA_2_106007 %{CISCOFW106006_106007_106010:details} +ASA_2_106010 %{CISCOFW106006_106007_106010} +ASA_3_106014 %{CISCOFW106014} +ASA_6_106015 %{CISCOFW106015} +ASA_1_106021 %{CISCOFW106021} +ASA_4_106023 %{CISCOFW106023} +ASA_5_106100 %{CISCOFW106100} +ASA_6_110002 %{CISCOFW110002} +ASA_6_302010 %{CISCOFW302010} +ASA_6_302013 %{CISCOFW302013_302014_302015_302016} +ASA_6_302014 %{CISCOFW302013_302014_302015_302016} +ASA_6_302015 %{CISCOFW302013_302014_302015_302016} +ASA_6_302016 %{CISCOFW302013_302014_302015_302016} +ASA_6_302020 %{CISCOFW302020_302021} +ASA_6_302021 %{CISCOFW302020_302021} +ASA_6_305011 %{CISCOFW305011} +ASA_3_313001 %{CISCOFW313001_313004_313008} +ASA_3_313004 %{CISCOFW313001_313004_313008} +ASA_3_313008 %{CISCOFW313001_313004_313008} +ASA_4_313005 %{CISCOFW313005} +ASA_4_402117 %{CISCOFW402117} +ASA_4_402119 %{CISCOFW402119} +ASA_4_419001 %{CISCOFW419001} +ASA_4_419002 %{CISCOFW419002} +ASA_4_500004 %{CISCOFW500004} +ASA_6_602303 %{CISCOFW602303_602304} +ASA_6_602304 %{CISCOFW602303_602304} +ASA_7_710001 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710002 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710003 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710005 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710006 %{CISCOFW710001_710002_710003_710005_710006} +ASA_6_713172 %{CISCOFW713172} +ASA_4_733100 %{CISCOFW733100} +ASA_6_305012 %{CISCOFW305012} +ASA_7_609001 %{CISCOFW609001} +ASA_7_609002 %{CISCOFW609002} +ASA_5_713041 %{CISCOFW713041} +ASA_3_305006 : %{CISCO_XLATE_CREATE_TYPE} translation creation failed for %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{DATA:err_src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{DATA:err_dst_fwuser}\))? + +#== End Cisco ASA == \ No newline at end of file diff --git a/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/samples/cisco_asa.txt b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/samples/cisco_asa.txt new file mode 100644 index 000000000..8e69dbd05 --- /dev/null +++ b/flink-cyber/cyber-jobs/src/main/resources/examples/pipelines/full/parse/samples/cisco_asa.txt @@ -0,0 +1,50 @@ +<166>Aug 05 2016 01:01:00 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.199/ssh for user 'root' +<166>Aug 05 2016 01:01:10 ASA : %ASA-6-302013: Built outbound TCP connection 82133946 for outside:203.0.113.70/443 to inside:192.168.1.49/53235 +<166>Aug 05 2016 01:01:20 ASA : %ASA-6-302013: Built outbound TCP connection 28488738 for outside:203.0.113.74/8080 to inside:192.168.1.139/15963 (session lasted 2509 seconds, 3 GB transferred) +<166>Aug 05 2016 01:01:30 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.48 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:01:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.35/27281 dst outside:203.0.113.86/10679 +<166>Aug 05 2016 01:01:50 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.185/33425 dst inside:203.0.113.78/43222 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:00 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.245/ssh for user 'user1' +<166>Aug 05 2016 01:02:10 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.207/6733 dst inside:203.0.113.136/24270 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:20 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.124 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:02:30 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.77 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:02:40 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.105/47867 dst inside:203.0.113.72/4011 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:02:50 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.237 dst inside:203.0.113.212 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:03:00 ASA : %ASA-6-302013: Built outbound TCP connection 71778197 for outside:203.0.113.60/8080 to inside:192.168.1.243/62767 (session lasted 1591 seconds, 10 GB transferred) +<166>Aug 05 2016 01:03:10 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.204 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:03:20 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.28 dst inside:203.0.113.118 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:03:30 ASA : %ASA-6-302013: Built outbound TCP connection 21235826 for outside:203.0.113.232/443 to inside:192.168.1.35/57987 +<166>Aug 05 2016 01:03:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.146/31215 dst outside:203.0.113.214/17544 +<166>Aug 05 2016 01:03:50 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.36/ssh for user 'root' +<166>Aug 05 2016 01:04:00 ASA : %ASA-6-302013: Built outbound TCP connection 65554300 for outside:203.0.113.235/443 to inside:192.168.1.153/35324 +<166>Aug 05 2016 01:04:10 ASA : %ASA-6-302013: Built outbound TCP connection 86288205 for outside:203.0.113.64/8080 to inside:192.168.1.225/24361 (session lasted 5995 seconds, 2 GB transferred) +<166>Aug 05 2016 01:04:20 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.19 dst inside:203.0.113.234 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:04:30 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.63/ssh for user 'user1' +<166>Aug 05 2016 01:04:40 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.186/ssh for user 'user1' +<166>Aug 05 2016 01:04:50 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.244/ssh for user 'user1' +<166>Aug 05 2016 01:05:00 ASA : %ASA-6-302013: Built outbound TCP connection 39234062 for outside:203.0.113.11/8080 to inside:192.168.1.89/20664 (session lasted 465 seconds, 6 GB transferred) +<166>Aug 05 2016 01:05:10 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.5/ssh for user 'admin' +<166>Aug 05 2016 01:05:20 ASA : %ASA-6-302013: Built outbound TCP connection 35055656 for outside:203.0.113.231/443 to inside:192.168.1.73/31031 +<166>Aug 05 2016 01:05:30 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.13/21008 dst inside:203.0.113.82/36378 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:05:40 ASA : %ASA-5-111010: Failed login attempt from 192.168.1.52/ssh for user 'user1' +<166>Aug 05 2016 01:05:50 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.114 dst inside:203.0.113.249 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:06:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.254/35812 dst inside:203.0.113.4/48884 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:06:10 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.94 dst inside:203.0.113.173 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:06:20 ASA : %ASA-6-302013: Built outbound TCP connection 82554922 for outside:203.0.113.75/443 to inside:192.168.1.201/44338 +<166>Aug 05 2016 01:06:30 ASA : %ASA-6-302013: Built outbound TCP connection 42701734 for outside:203.0.113.222/443 to inside:192.168.1.205/21435 +<166>Aug 05 2016 01:06:40 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.195/40493 dst inside:203.0.113.205/62028 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:06:50 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.79/45112 dst inside:203.0.113.63/62563 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.35/21904 dst inside:203.0.113.219/50940 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:10 ASA : %ASA-6-302013: Built outbound TCP connection 49890331 for outside:203.0.113.10/8080 to inside:192.168.1.87/51767 (session lasted 3877 seconds, 6 GB transferred) +<166>Aug 05 2016 01:07:20 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.208 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:07:30 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.6/25226 dst inside:203.0.113.49/15638 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:07:40 ASA : %ASA-6-302013: Built outbound TCP connection 42193182 for outside:203.0.113.197/443 to inside:192.168.1.246/58966 +<166>Aug 05 2016 01:07:50 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.234 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:08:00 ASA : %ASA-6-302013: Built outbound TCP connection 49955429 for outside:203.0.113.21/443 to inside:192.168.1.96/58844 +<166>Aug 05 2016 01:08:10 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.194/30591 dst inside:203.0.113.226/19073 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:08:20 ASA : %ASA-6-302013: Built outbound TCP connection 89392405 for outside:203.0.113.165/8080 to inside:192.168.1.130/37276 (session lasted 568 seconds, 6 GB transferred) +<166>Aug 05 2016 01:08:30 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.195 dst inside:203.0.113.155 (type 3, code 3) on outside interface. +<166>Aug 05 2016 01:08:40 ASA : %ASA-3-305006: portmap translation creation failed for udp src inside:192.168.1.142/8061 dst outside:203.0.113.28/25227 +<166>Aug 05 2016 01:08:50 ASA : %ASA-4-400013: IDS:2004 ICMP PING SWEEP from 192.168.1.146 on interface outside. Potentially a reconnaissance attempt. +<166>Aug 05 2016 01:09:00 ASA : %ASA-4-106023: Deny tcp src outside:192.168.1.191/59896 dst inside:203.0.113.241/51735 by access-group "outside_access_in" [0x0, 0x0] +<166>Aug 05 2016 01:09:10 ASA : %ASA-4-313005: No matching connection for ICMP error message: icmp src outside:192.168.1.12 dst inside:203.0.113.232 (type 3, code 3) on outside interface. diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokTableParser.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokTableParser.java new file mode 100644 index 000000000..e5886ed3c --- /dev/null +++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokTableParser.java @@ -0,0 +1,298 @@ +package com.cloudera.parserchains.parsers; +/* + * Copyright 2020 - 2022 Cloudera. All Rights Reserved. + * + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and + * limitations governing your use of the file. + */ + +import com.cloudera.parserchains.core.*; +import com.cloudera.parserchains.core.catalog.Configurable; +import com.cloudera.parserchains.core.catalog.MessageParser; +import com.cloudera.parserchains.core.catalog.Parameter; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.krakens.grok.api.Grok; +import io.krakens.grok.api.GrokCompiler; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.lang.String.format; + +@Slf4j +@MessageParser( + name = "Grok Table", + description = "Extracts a key using an initial grok expression and parses the message using a grok expression mapped from a table." +) +public class GrokTableParser implements Parser { + public static final String KEY_FIELD_NAME_CONFIGURATION = "Key Field Name"; + public static final String MESSAGE_FIELD_NAME_CONFIGURATION = "Message Field Name"; + public static final String INITIAL_GROK_EXPRESSION_CONFIGURATION = "Initial Grok Expression"; + public static final String GROK_PATTERN_PATH_CONFIGURATION = "Grok Pattern Path"; + public static final String INPUT_FIELD_CONFIGURATION = "Input Field"; + public static final String MISSING_REQUIRED_CONFIGURATION_FIELD_ERROR_FORMAT = "Missing required configuration field(s): %s"; + public static final String PATTERN_LOADING_ERROR_FORMAT = "Pattern files '%s' could not be loaded"; + public static final String MESSAGE_TO_PARSE_EXPECTED_INPUT_FIELD_ERROR_FORMAT = "Message to parse expected input field '%s'"; + public static final String FAILED_MESSAGE_GROK_COMPILATION_ERROR_FORMAT = "Grok compilation error in message '%s': '%s'"; + public static final String MESSAGE_KEY_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT = "Message key field '%s' is null after grok capture"; + public static final String MESSAGE_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT = "Message field '%s' is null after grok capture"; + + enum ValidationState { + UNVALIDATED, + VALID, + INVALID + } + + private FieldName inputField; + private final GrokCompiler grokCompiler; + private Grok initialGrokExpression; + private String keyFieldName; + private String messageFieldName; + private static class GrokCompilerResult { + Grok compiledGrok; + String errorMessage; + + private GrokCompilerResult(Grok compiledGrok, String errorMessage) { + this.compiledGrok = compiledGrok; + this.errorMessage = errorMessage; + } + + static GrokCompilerResult create(String grokExpression, GrokCompiler grokCompiler) { + Grok compiledGrok = null; + String errorMessage = null; + try { + compiledGrok = grokCompiler.compile(grokExpression, true); + } catch (Exception e) { + errorMessage = e.getMessage(); + } + return new GrokCompilerResult(compiledGrok, errorMessage); + } + } + private final Cache grokTable; + private ValidationState isValid = ValidationState.UNVALIDATED; + private String validationError = null; + private final Map patternsLoaded = new HashMap<>(); + + public GrokTableParser() { + inputField = FieldName.of(Constants.DEFAULT_INPUT_FIELD); + grokCompiler = GrokCompiler.newInstance(); + grokCompiler.registerDefaultPatterns(); + initialGrokExpression = null; + keyFieldName = null; + messageFieldName = null; + grokTable = Caffeine.newBuilder() + .maximumSize(100) + .build(); + } + + @Override + public Message parse(Message input) { + Message.Builder output = Message.builder().withFields(input); + if (!isConfigured()) { + output.withError(validationError); + } else if (!input.getField(inputField).isPresent()) { + output.withError(format(MESSAGE_TO_PARSE_EXPECTED_INPUT_FIELD_ERROR_FORMAT, inputField.toString())); + } else { + input.getField(inputField).ifPresent(val -> doParse(val.toString(), output)); + } + return output.build(); + } + + private boolean isConfigured() { + if (isValid == ValidationState.UNVALIDATED) { + List missingFields = new ArrayList<>(); + checkIfNonBlank(keyFieldName, KEY_FIELD_NAME_CONFIGURATION, missingFields); + checkIfNonBlank(messageFieldName, MESSAGE_FIELD_NAME_CONFIGURATION, missingFields); + if (initialGrokExpression == null) { + missingFields.add(INITIAL_GROK_EXPRESSION_CONFIGURATION); + } + if (patternsLoaded.isEmpty()) { + missingFields.add((GROK_PATTERN_PATH_CONFIGURATION)); + } + if (!missingFields.isEmpty()) { + validationError = String.format(MISSING_REQUIRED_CONFIGURATION_FIELD_ERROR_FORMAT, StringUtils.join(missingFields, ", ")); + isValid = ValidationState.INVALID; + } else if (patternsLoaded.entrySet().stream().anyMatch(e -> !e.getValue())) { + String failedPatternFiles = patternsLoaded.entrySet().stream() + .filter(entry -> !entry.getValue()) // Filter where value is false + .map(Map.Entry::getKey) // Get the keys + .collect(Collectors.joining(", ")); + validationError = String.format(PATTERN_LOADING_ERROR_FORMAT,failedPatternFiles); + isValid = ValidationState.INVALID; + } else { + isValid = ValidationState.VALID; + } + } + return (isValid == ValidationState.VALID); + } + + private void checkIfNonBlank(String configuredValue, String fieldName, List missingFields ) { + if (StringUtils.isBlank(configuredValue)) { + missingFields.add(fieldName); + } + } + + private void doParse(String textToParse, Message.Builder output) { + Map grokResult = initialGrokExpression.match(textToParse) + .capture() + .entrySet() + .stream() + .filter(e -> e.getKey() != null && e.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + String key = getGrokString(grokResult, keyFieldName); + String message = getGrokString(grokResult, messageFieldName); + // add the base message fields to the output + grokResult.forEach((field_name, field_value) -> output.addField(field_name, field_value.toString())); + if (key == null) { + output.withError(format(MESSAGE_KEY_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT, keyFieldName)); + } else if (message == null) { + output.withError(format(MESSAGE_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT, messageFieldName)); + } else { + doMessageGrok(key, message, output); + } + } + + private String getGrokString(Map grokResult, String fieldName) { + Object valueObject = grokResult.get(fieldName); + String valueString = null; + + if (valueObject != null) { + valueString = valueObject.toString(); + if (StringUtils.isBlank(valueString)) { + valueString = null; + } + } + + return valueString; + } + + private void doMessageGrok(String messageKey, String textToParse, Message.Builder output) { + String sanitizedMessageKey = messageKey.replaceAll("[^A-z0-9_]+", "_"); + String messagePattern = grokCompiler.getPatternDefinitions().get(sanitizedMessageKey); + // if there is a pattern, defined for this message key, capture the pattern + // if not, just return the fields extracted by the initial grok expression + if (messagePattern != null) { + GrokCompilerResult grokResult = grokTable.get(messagePattern, pattern -> GrokCompilerResult.create(messagePattern, grokCompiler)); + if (grokResult != null) { + if (grokResult.compiledGrok != null) { + grokResult.compiledGrok.match(textToParse) + .capture() + .entrySet() + .stream() + .filter(e -> e.getKey() != null && e.getValue() != null) + .forEach(e -> output.addField(e.getKey(), e.getValue().toString())); + } else { + output.withError(format(FAILED_MESSAGE_GROK_COMPILATION_ERROR_FORMAT, sanitizedMessageKey, grokResult.errorMessage)); + } + } + } + } + + @Configurable(key = "keyFieldName", + label = KEY_FIELD_NAME_CONFIGURATION, + description = "The name of the key field extracted from the initialGrokExpression.", + required = true) + public GrokTableParser keyFieldName(String keyFieldName) { + + if (StringUtils.isNoneBlank(keyFieldName)) { + this.keyFieldName = keyFieldName; + } + + return this; + } + + @Configurable(key = "messageFieldName", + label = MESSAGE_FIELD_NAME_CONFIGURATION, + description = "The name of the message field extracted from the initialGrokExpression.", + required = true) + public GrokTableParser messageFieldName(String messageFieldName) { + + if (StringUtils.isNoneBlank(messageFieldName)) { + this.messageFieldName = messageFieldName; + } + + return this; + } + + @Configurable(key = "initialGrokExpression", + label = INITIAL_GROK_EXPRESSION_CONFIGURATION, + description = "The initial grok expression used to extract the key and message.", + required = true) + public GrokTableParser initialGrokExpression(String initialGrokExpression) { + if (StringUtils.isNotBlank(initialGrokExpression)) { + this.initialGrokExpression = grokCompiler.compile(initialGrokExpression, true); + } + return this; + } + + @Configurable( + key = "grokPatternPath", + label = GROK_PATTERN_PATH_CONFIGURATION, + description = "Path to file containing, grok patterns used in key grok expression and mapping message keys to expressions. Define pattern name then whitespace followed by pattern expression. Define a pattern for each message type that you want to extract. Replace all non alphanumeric characters with underscores.", + orderPriority = 1) + public GrokTableParser grokPatternPath(@Parameter(key = "grokPatternPath", isPath = true, + label = GROK_PATTERN_PATH_CONFIGURATION, + description = "Path to file containing, grok patterns used in key grok expression and mapping message keys to expressions. Define pattern name then whitespace followed by pattern expression. Define a pattern for each message type that you want to extract. Replace all non alphanumeric characters with underscores.", + required = true) String grokPatternPath) { + if (StringUtils.isNotBlank(grokPatternPath)) { + try { + patternsLoaded.put(grokPatternPath, false); + FileSystem fileSystem = new Path(grokPatternPath).getFileSystem(); + Path path = new Path(grokPatternPath); + log.info("Loading grok patterns from {}", path); + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileSystem.open(path)))) { + this.grokCompiler.register(bufferedReader); + patternsLoaded.put(grokPatternPath, true); + } + catch (IOException e) { + log.error("Failed to load grok patterns from {}", path, e); + throw new IllegalArgumentException("Invalid path while load grok patterns from " + grokPatternPath, e); + } + } catch (Exception e) { + log.error("Invalid path while load grok patterns from {}", grokPatternPath, e); + throw new IllegalArgumentException("Invalid path while load grok patterns from " + grokPatternPath, e); + } + } + + return this; + } + + public GrokTableParser inputField(FieldName inputField) { + if (inputField != null && StringUtils.isNotBlank(inputField.get())) { + this.inputField = inputField; + } + return this; + } + + @Configurable(key = "inputField", + label = INPUT_FIELD_CONFIGURATION, + description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'", + defaultValue = Constants.DEFAULT_INPUT_FIELD) + public GrokTableParser inputField(String inputField) { + if (StringUtils.isNotBlank(inputField)) { + this.inputField = FieldName.of(inputField); + } + return this; + } + + public FieldName getInputField() { + return inputField; + } + +} diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokTableParserTest.java b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokTableParserTest.java new file mode 100644 index 000000000..5e2327df8 --- /dev/null +++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/java/com/cloudera/parserchains/parsers/GrokTableParserTest.java @@ -0,0 +1,369 @@ +/* + * Copyright 2020 - 2022 Cloudera. All Rights Reserved. + * + * This file is licensed under the Apache License Version 2.0 (the "License"). You may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. Refer to the License for the specific permissions and + * limitations governing your use of the file. + */ + +package com.cloudera.parserchains.parsers; + +import com.cloudera.parserchains.core.Constants; +import com.cloudera.parserchains.core.FieldName; +import com.cloudera.parserchains.core.Message; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.*; + + +public class GrokTableParserTest { + private static final String CISCO_LOG_MESSAGE = "<164>Aug 05 2016 01:01:34: %ASA-4-106023: Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]"; + private static final String UNMAPPED_CISCO_LOG_MESSAGE = "<164>Aug 05 2016 01:01:34: %ASA-1-123456: Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]"; + private static final String CISCO_CONNECTION_MESSAGE = "<174>Jan 5 14:52:35 10.22.8.212 %ASA-6-302015: Built inbound UDP connection 76245506 for outside:10.22.8.110/49886 (10.22.8.110/49886) to inside:192.111.72.8/8612 (192.111.72.8/8612) (user.name)"; + private static final String CISCO_MESSAGE_GROK_PATTERNS_PATH = "/grok/cisco_asa"; + private static final String INPUT_FIELD_NAME = "input"; + private static final String CISCO_INITIAL_GROK_EXPRESSION = "%{CISCO_TAGGED_SYSLOG}"; + private static final String CISCO_KEY_FIELD_NAME = "cisco_tag"; + private static final String CISCO_MESSAGE_FIELD_NAME = "message"; + + private GrokTableParser grokTableParser; + + @BeforeEach + void beforeEach() { + grokTableParser = new GrokTableParser(); + } + + @Test + void testValidConfigurationWithDefaultInput() { + testSuccessfulMessages(Constants.DEFAULT_INPUT_FIELD); + } + + @Test + void testValidConfigurationWithInputField() { + testSuccessfulMessages(INPUT_FIELD_NAME); + } + + @Test + void testEmptyMessage() { + String messageWithEmptyText = "%ASA-4-106023: "; + + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + + grokTableParser + .inputField(FieldName.of(INPUT_FIELD_NAME)) + .grokPatternPath(patternPath) + .initialGrokExpression("%{CISCOTAG:cisco_tag}:%{GREEDYDATA:message}") + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + Message messageMissingInput = Message.builder(). + addField(INPUT_FIELD_NAME, messageWithEmptyText) + .build(); + Message output = grokTableParser.parse(messageMissingInput); + + checkErrorMessage(output, String.format(GrokTableParser.MESSAGE_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT, CISCO_MESSAGE_FIELD_NAME)); + } + + private void testSuccessfulMessages(String inputFieldName) { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + + grokTableParser + .grokPatternPath(patternPath) + .initialGrokExpression(CISCO_INITIAL_GROK_EXPRESSION) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + if (!Constants.DEFAULT_INPUT_FIELD.equals(inputFieldName)) { + grokTableParser.inputField(inputFieldName); + } + + Message expectedParsedDenyMessage = Message.builder() + .addField("timestamp", "Aug 05 2016 01:01:34") + .addField(inputFieldName, CISCO_LOG_MESSAGE) + .addField("action", "Deny") + .addField("src_ip", "10.30.9.121") + .addField("hashcode1", "0x962df600") + .addField("hashcode2", "0x0") + .addField("dst_interface", "Outside") + .addField("protocol", "tcp") + .addField("src_interface", "Inside") + .addField("policy_id", "Inside_access_in") + .addField("dst_port", "42028") + .addField("syslog_pri", "164") + .addField("message", ": Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]") + .addField("dst_ip", "192.168.135.51") + .addField("src_port", "54580") + .addField("cisco_tag", "ASA-4-106023") + .build(); + + testMessageParse(inputFieldName, CISCO_LOG_MESSAGE, expectedParsedDenyMessage); + + Message expectedParsedConnectionMessage = Message.builder() + .addField("timestamp", "Jan 5 14:52:35") + .addField(inputFieldName, CISCO_CONNECTION_MESSAGE) + .addField("action", "Built") + .addField("src_ip", "10.22.8.110") + .addField("dst_interface", "inside") + .addField("dst_mapped_ip", "192.111.72.8") + .addField("protocol", "UDP").addField("src_interface", "outside") + .addField("dst_port", "8612") + .addField("syslog_pri", "174") + .addField("direction", "inbound") + .addField("syslog_host", "10.22.8.212") + .addField("message", ": Built inbound UDP connection 76245506 for outside:10.22.8.110/49886 (10.22.8.110/49886) to inside:192.111.72.8/8612 (192.111.72.8/8612) (user.name)") + .addField("src_mapped_ip", "10.22.8.110") + .addField("connection_id", "76245506") + .addField("dst_ip", "192.111.72.8") + .addField("src_mapped_port", "49886") + .addField("dst_mapped_port", "8612") + .addField("reason", "") + .addField("src_port", "49886") + .addField("cisco_tag", "ASA-6-302015") + .build(); + testMessageParse(inputFieldName, CISCO_CONNECTION_MESSAGE, expectedParsedConnectionMessage); + } + + private void testMessageParse(String inputFieldName, String messageToParse, Message expected) { + Message input = Message.builder() + .addField(inputFieldName, messageToParse) + .build(); + Message output = grokTableParser.parse(input); + + assertThat("Parsed message fields do not match expected values.", + output, is(expected)); + assertFalse(output.getError().isPresent(), "Did not expect errors in parsing"); + } + + @Test + public void testBadGrokExpressionInPath() { + String patternPath = getFileFromResource("/grok/bad_grok_pattern").getAbsolutePath(); + + grokTableParser + .grokPatternPath(patternPath) + .initialGrokExpression("%{INT:message_id} : %{GREEDYDATA:message}") + .keyFieldName("message_id") + .messageFieldName("message"); + + Message output = grokTableParser.parse(Message.builder().addField(grokTableParser.getInputField().get(), "1234 : this is a test").build()); + checkErrorMessage(output, String.format(GrokTableParser.FAILED_MESSAGE_GROK_COMPILATION_ERROR_FORMAT, "1234", "Illegal repetition near index 0\n%{INT\n^")); + } + + @Test + public void testBadInitialGrokExpression() { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + + grokTableParser + .grokPatternPath(patternPath) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + try { + grokTableParser.initialGrokExpression("%{UNDEFINED_GROK_PATTERN}"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException iae) { + Message output = grokTableParser.parse(Message.builder().addField(grokTableParser.getInputField().get(), "1234 : this is a test").build()); + checkErrorMessage(output, String.format(GrokTableParser.MISSING_REQUIRED_CONFIGURATION_FIELD_ERROR_FORMAT, GrokTableParser.INITIAL_GROK_EXPRESSION_CONFIGURATION)); + } + + } + + @Test + public void testNoMatchingExpressionForMessageKeyReturnsInitialGrokFields() { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + + grokTableParser + .grokPatternPath(patternPath) + .initialGrokExpression(CISCO_INITIAL_GROK_EXPRESSION) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + + Message expectedMessage = Message.builder() + .addField("timestamp", "Aug 05 2016 01:01:34") + .addField(grokTableParser.getInputField().get(), UNMAPPED_CISCO_LOG_MESSAGE) + .addField("syslog_pri", "164") + .addField("message", ": Deny tcp src Inside:10.30.9.121/54580 dst Outside:192.168.135.51/42028 by access-group \"Inside_access_in\" [0x962df600, 0x0]") + .addField("cisco_tag", "ASA-1-123456") + .build(); + + testMessageParse(grokTableParser.getInputField().get(), UNMAPPED_CISCO_LOG_MESSAGE, expectedMessage); + } + + @Test + public void testMissingKeyFieldValue() { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + String fieldNotInGrok = "field_not_in_grok"; + grokTableParser + .inputField(FieldName.of(INPUT_FIELD_NAME)) + .grokPatternPath(patternPath) + .initialGrokExpression(CISCO_INITIAL_GROK_EXPRESSION) + .keyFieldName(fieldNotInGrok) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + Message messageMissingInput = Message.builder(). + addField(INPUT_FIELD_NAME, CISCO_LOG_MESSAGE) + .build(); + Message output = grokTableParser.parse(messageMissingInput); + + checkErrorMessage(output, String.format(GrokTableParser.MESSAGE_KEY_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT, fieldNotInGrok)); + } + + @Test + public void testMissingMessageFieldValue() { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + String fieldNotInGrok = "field_not_in_grok"; + grokTableParser + .inputField(FieldName.of(INPUT_FIELD_NAME)) + .grokPatternPath(patternPath) + .initialGrokExpression(CISCO_INITIAL_GROK_EXPRESSION) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(fieldNotInGrok); + + Message messageMissingInput = Message.builder(). + addField(INPUT_FIELD_NAME, CISCO_LOG_MESSAGE) + .build(); + Message output = grokTableParser.parse(messageMissingInput); + + checkErrorMessage(output, String.format(GrokTableParser.MESSAGE_FIELD_NULL_AFTER_GROK_CAPTURE_ERROR_FORMAT, fieldNotInGrok)); + } + + + @Test + public void testInputMessageMissingFromMessageToParse() { + String patternPath = getFileFromResource(CISCO_MESSAGE_GROK_PATTERNS_PATH).getAbsolutePath(); + + grokTableParser + .inputField(FieldName.of(INPUT_FIELD_NAME)) + .grokPatternPath(patternPath) + .initialGrokExpression(CISCO_INITIAL_GROK_EXPRESSION) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME); + + Message messageMissingInput = Message.builder(). + addField("wrong_input_field_name", "Message won' be parsed") + .build(); + Message output = grokTableParser.parse(messageMissingInput); + + checkErrorMessage(output, String.format(GrokTableParser.MESSAGE_TO_PARSE_EXPECTED_INPUT_FIELD_ERROR_FORMAT, INPUT_FIELD_NAME)); + } + + @Test + public void testMissingKeyFieldName() { + testParserConfigError(null, CISCO_MESSAGE_FIELD_NAME, CISCO_INITIAL_GROK_EXPRESSION, CISCO_MESSAGE_GROK_PATTERNS_PATH); + } + + @Test + public void testMissingMessageFieldName() { + testParserConfigError(CISCO_KEY_FIELD_NAME, null, CISCO_INITIAL_GROK_EXPRESSION, CISCO_MESSAGE_GROK_PATTERNS_PATH); + } + + @Test + public void testMissingInitialGrokExpression() { + testParserConfigError(CISCO_KEY_FIELD_NAME, CISCO_MESSAGE_FIELD_NAME, null, CISCO_MESSAGE_GROK_PATTERNS_PATH); + } + + @Test + public void testMissingGrokPatternPath() { + testParserConfigError(CISCO_KEY_FIELD_NAME, CISCO_MESSAGE_FIELD_NAME, "%{GREEDYDATA}", null); + } + + @Test + public void testGrokPathDoesNotExist() { + String pathDoesNotExist = "/bad_path"; + testInvalidPath(pathDoesNotExist); + } + + @Test + public void testMalformedGrokPath() { + String malformedPath = "malformed:path"; + testInvalidPath(malformedPath); + } + + @Test + public void testNullEmptyInputFieldName() { + grokTableParser.inputField((String) null); + assertEquals(Constants.DEFAULT_INPUT_FIELD, grokTableParser.getInputField().get()); + + grokTableParser.inputField(" "); + assertEquals(Constants.DEFAULT_INPUT_FIELD, grokTableParser.getInputField().get()); + + FieldName fieldName = FieldName.of(" "); + grokTableParser.inputField(fieldName); + assertEquals(Constants.DEFAULT_INPUT_FIELD, grokTableParser.getInputField().get()); + + grokTableParser.inputField((FieldName) null); + assertEquals(Constants.DEFAULT_INPUT_FIELD, grokTableParser.getInputField().get()); + } + + private void testInvalidPath(String pathDoesNotExist) { + grokTableParser.inputField(FieldName.of(INPUT_FIELD_NAME)) + .keyFieldName(CISCO_KEY_FIELD_NAME) + .messageFieldName(CISCO_MESSAGE_FIELD_NAME) + .initialGrokExpression("%{GREEDYDATA}"); + try { + grokTableParser.grokPatternPath(pathDoesNotExist); + fail("Expected illegal argument exception due to invalid grok pattern path."); + } catch (IllegalArgumentException e) { + Message input = Message.builder() + .addField(INPUT_FIELD_NAME, CISCO_LOG_MESSAGE) + .build(); + + Message output = grokTableParser.parse(input); + checkErrorMessage(output, String.format(GrokTableParser.PATTERN_LOADING_ERROR_FORMAT, pathDoesNotExist)); + } + } + + private void testParserConfigError(String keyField, String messageFieldName, String initialGrokExpression, String grokPatternPath) { + + if (grokPatternPath != null) { + grokPatternPath = getFileFromResource("/grok/cisco_asa").getAbsolutePath(); + } + Message input = Message.builder() + .addField(grokTableParser.getInputField(), CISCO_LOG_MESSAGE) + .build(); + + Message output = grokTableParser + .grokPatternPath(grokPatternPath) + .initialGrokExpression(initialGrokExpression) + .keyFieldName(keyField) + .messageFieldName(messageFieldName) + .parse(input); + + List expectedMissingFields = new ArrayList<>(); + addMissingConfigField(GrokTableParser.KEY_FIELD_NAME_CONFIGURATION, keyField, expectedMissingFields); + addMissingConfigField(GrokTableParser.MESSAGE_FIELD_NAME_CONFIGURATION, messageFieldName, expectedMissingFields); + addMissingConfigField(GrokTableParser.INITIAL_GROK_EXPRESSION_CONFIGURATION, initialGrokExpression, expectedMissingFields); + addMissingConfigField(GrokTableParser.GROK_PATTERN_PATH_CONFIGURATION, grokPatternPath, expectedMissingFields); + + String expectedErrorMessage = String.format(GrokTableParser.MISSING_REQUIRED_CONFIGURATION_FIELD_ERROR_FORMAT, StringUtils.join(expectedMissingFields, ", ")); + checkErrorMessage(output, expectedErrorMessage); + } + + private static void checkErrorMessage(Message output, String expectedErrorMessage) { + assertTrue(output.getError().isPresent(), "Expected errors in parsing"); + assertThat(output.getError().get().getMessage(), is(expectedErrorMessage)); + } + + private void addMissingConfigField(String configName, String configFieldValue, List missingFields) { + if (configFieldValue == null) { + missingFields.add(configName); + } + } + + private static File getFileFromResource(String path) { + return new File(Objects.requireNonNull(GrokTableParser.class.getResource(path)).getFile()); + } +} diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/bad_grok_pattern b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/bad_grok_pattern new file mode 100644 index 000000000..0d8b70509 --- /dev/null +++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/bad_grok_pattern @@ -0,0 +1 @@ +1234 %{INT \ No newline at end of file diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/cisco_asa b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/cisco_asa new file mode 100644 index 000000000..8892ae3cf --- /dev/null +++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/src/test/resources/grok/cisco_asa @@ -0,0 +1,220 @@ +# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME:UNWANTED} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM:UNWANTED}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,~-]+|\\.)*)+ +#UNIXPATH (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port:int})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +# Time: HH:MM:SS +#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)? +# I'm still on the fence about using grok to perform the time match, +# since it's probably slower. +# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)? +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +GREEDYDATA .* + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid:int}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility:int}.%{NONNEGINT:priority:int}> +SYSLOGPRIORITY <%{POSINT:syslog_pri:int}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING:UNWANTED} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: + +MESSAGESLOG %{SYSLOGBASE} %{DATA} + +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} + +# Log Levels +LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + +#== Cisco ASA == +CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri:int}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:syslog_host})?( %{SYSLOGPROG:syslog_prog})? ?:? %%{CISCOTAG:cisco_tag}%{GREEDYDATA:message} +CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} +CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) + +# Common Particles +CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted +CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* +CISCO_DIRECTION Inbound|inbound|Outbound|outbound +CISCO_INTERVAL first hit|%{INT}-second interval +CISCO_XLATE_TYPE static|dynamic +# ASA-2-106001 +CISCOFW106001 : %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-2-106006, ASA-2-106007, ASA-2-106010 +CISCOFW106006_106007_106010 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port:int}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port:int}(\(%{DATA:dst_fwuser}\))? (?:on interface %{GREEDYDATA:interface}|due to %{CISCO_REASON:reason}) +# ASA-3-106014 +CISCOFW106014 : %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type:int}, code %{INT:icmp_code:int}\) +# ASA-6-106015 +CISCOFW106015 : %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} +# ASA-1-106021 +CISCOFW106021 : %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} +# ASA-4-106023 +CISCOFW106023 : %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-5-106100 +CISCOFW106100 : access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port:int}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port:int}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count:int} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] +# ASA-6-110002 +CISCOFW110002 : %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-302010 +CISCOFW302010 : %{INT:connection_count:int} in use, %{INT:connection_count_max:int} most used +# ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 +CISCOFW302013_302014_302015_302016 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id:int} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port:int}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port:int}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes:int})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? +# ASA-6-302020, ASA-6-302021 +CISCOFW302020_302021 : %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? +# ASA-6-305011 +CISCOFW305011 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} +# ASA-3-313001, ASA-3-313004, ASA-3-313008 +CISCOFW313001_313004_313008 : %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type:int}, code=%{INT:icmp_code:int} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? +# ASA-4-313005 +CISCOFW313005 : %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type:int}, code %{INT:err_icmp_code:int}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port:int}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port:int}(\(%{DATA:orig_dst_fwuser}\))? +# ASA-4-402117 +CISCOFW402117 : %{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} +# ASA-4-402119 +CISCOFW402119 : %{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking +# ASA-4-419001 +CISCOFW419001 : %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int}, reason: %{GREEDYDATA:reason} +# ASA-4-419002 +CISCOFW419002 : %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} with different initial sequence number +# ASA-4-500004 +CISCOFW500004 : %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port:int} to %{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-602303, ASA-6-602304 +CISCOFW602303_602304 : %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} +# ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 +CISCOFW710001_710002_710003_710005_710006 : %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port:int} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port:int} +# ASA-6-713172 +CISCOFW713172 : Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device +# ASA-4-733100 +CISCOFW733100 : \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst:int} per second, max configured rate is %{INT:drop_rate_max_burst:int}; Current average rate is %{INT:drop_rate_current_avg:int} per second, max configured rate is %{INT:drop_rate_max_avg:int}; Cumulative total count is %{INT:drop_total_count:int} + + +# ASA-6-305012 +CISCOFW305012 : %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} duration %{TIME:duration} +# ASA-7-609001 +CISCOFW609001 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? +# ASA-7-609002 +CISCOFW609002 : %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port:int})?(\(%{DATA:src_fwuser}\))? duration %{TIME:duration} + +# ASA-5-713041 +CISCOFW713041 : (Group = %{GREEDYDATA:group}, )?IP = %{IP:src_ip}, IKE Initiator: Rekeying Phase %{INT}, Intf %{DATA:src_interface}, IKE Peer %{DATA}\s+local Proxy Address %{IP}, remote Proxy Address %{IP},\s+Crypto map \(%{DATA}\) + +ASA_2_106001 %{CISCOFW106001:details} +ASA_2_106006 %{CISCOFW106006_106007_106010:details} +ASA_2_106007 %{CISCOFW106006_106007_106010:details} +ASA_2_106010 %{CISCOFW106006_106007_106010} +ASA_3_106014 %{CISCOFW106014} +ASA_6_106015 %{CISCOFW106015} +ASA_1_106021 %{CISCOFW106021} +ASA_4_106023 %{CISCOFW106023} +ASA_5_106100 %{CISCOFW106100} +ASA_6_110002 %{CISCOFW110002} +ASA_6_302010 %{CISCOFW302010} +ASA_6_302013 %{CISCOFW302013_302014_302015_302016} +ASA_6_302014 %{CISCOFW302013_302014_302015_302016} +ASA_6_302015 %{CISCOFW302013_302014_302015_302016} +ASA_6_302016 %{CISCOFW302013_302014_302015_302016} +ASA_6_302020 %{CISCOFW302020_302021} +ASA_6_302021 %{CISCOFW302020_302021} +ASA_6_305011 %{CISCOFW305011} +ASA_3_313001 %{CISCOFW313001_313004_313008} +ASA_3_313004 %{CISCOFW313001_313004_313008} +ASA_3_313008 %{CISCOFW313001_313004_313008} +ASA_4_313005 %{CISCOFW313005} +ASA_4_402117 %{CISCOFW402117} +ASA_4_402119 %{CISCOFW402119} +ASA_4_419001 %{CISCOFW419001} +ASA_4_419002 %{CISCOFW419002} +ASA_4_500004 %{CISCOFW500004} +ASA_6_602303 %{CISCOFW602303_602304} +ASA_6_602304 %{CISCOFW602303_602304} +ASA_7_710001 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710002 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710003 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710005 %{CISCOFW710001_710002_710003_710005_710006} +ASA_7_710006 %{CISCOFW710001_710002_710003_710005_710006} +ASA_6_713172 %{CISCOFW713172} +ASA_4_733100 %{CISCOFW733100} +ASA_6_305012 %{CISCOFW305012} +ASA_7_609001 %{CISCOFW609001} +ASA_7_609002 %{CISCOFW609002} +ASA_5_713041 %{CISCOFW713041} + +#== End Cisco ASA == \ No newline at end of file