description: Pipeline for parsing fortinet firewall logs
processors:
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- grok:
    field: message
    patterns:
    - '%{SYSLOG5424PRI}%{GREEDYDATA:syslog5424_sd}$'
- gsub:
    field: syslog5424_sd
    pattern: "\u0000"
    replacement: ""
    if: ctx.syslog5424_sd != null
- script:
    lang: painless
    if: ctx.syslog5424_sd != null
    description: |
      Splits syslog5424_sd KV list by space and then each by "=" taking into account quoted values.
    source:
      def splitUnquoted(String input, String sep) {
        def tokens = [];
        def startPosition = 0;
        def isInQuotes = false;
        char quote = (char)"\"";
        for (def currentPosition = 0; currentPosition < input.length(); currentPosition++) {
            if (input.charAt(currentPosition) == quote) {
                isInQuotes = !isInQuotes;
            }
            else if (input.charAt(currentPosition) == (char)sep && !isInQuotes) {
                def token = input.substring(startPosition, currentPosition).trim();
                if (!token.equals("")) {
                  tokens.add(token);
                }
                startPosition = currentPosition + 1;
            }
        }

        def lastToken = input.substring(startPosition);
        if (!lastToken.equals(sep) && !lastToken.equals("")) {
            tokens.add(lastToken.trim());
        }
        return tokens;
      }

      def arr = splitUnquoted(ctx.syslog5424_sd, " ");

      Map map = new HashMap();
      Pattern pattern = /^\"|\"$/;
      for (def i = 0; i < arr?.length; i++) {
        def kv = splitUnquoted(arr[i], "=");
        if (kv.length == 2) {
          map[kv[0]] = pattern.matcher(kv[1]).replaceAll("");
        }
      }
      if (ctx.fortinet == null) {
        ctx.fortinet = new HashMap();
      }
      ctx.fortinet.firewall = map;
- script:
    lang: painless
    source: |
      def fw = ctx?.fortinet?.firewall;
      if (fw != null) {
        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A" || entry.getValue() == "undefined");
      }
- set:
    field: observer.vendor
    value: Fortinet
- set:
    field: observer.product
    value: Fortigate
- set:
    field: observer.type
    value: firewall
- set:
    field: event.timezone
    value: "{{fortinet.firewall.tz}}"
    ignore_empty_value: true
- set:
    field: _temp.time
    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{event.timezone}}"
    if: "ctx.fortinet?.firewall?.date != null && ctx.fortinet?.firewall?.time != null && ctx.event?.timezone != null"
- set:
    field: _temp.time
    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}"
    if: "ctx.fortinet?.firewall?.date != null && ctx.fortinet?.firewall?.time != null && ctx.event?.timezone == null"
- date:
    field: _temp.time
    target_field: "@timestamp"
    formats:
    - yyyy-MM-dd HH:mm:ss
    - yyyy-MM-dd HH:mm:ss Z
    - yyyy-MM-dd HH:mm:ss z
    - ISO8601
    timezone: "{{event.timezone}}"
    if: "ctx._temp?.time != null && ctx.event?.timezone != null"
- date:
    field: _temp.time
    target_field: "@timestamp"
    formats:
    - yyyy-MM-dd HH:mm:ss
    - yyyy-MM-dd HH:mm:ss Z
    - yyyy-MM-dd HH:mm:ss z
    - ISO8601
    if: "ctx._temp?.time != null && ctx.event?.timezone == null"
- gsub:
    field: fortinet.firewall.eventtime
    pattern: "\\d{6}$"
    replacement: ""
    if: "ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 18"
- date:
    field: fortinet.firewall.eventtime
    target_field: event.start
    formats:
    - UNIX_MS
    timezone: "{{event.timezone}}"
    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
- date:
    field: fortinet.firewall.eventtime
    target_field: event.start
    formats:
    - UNIX
    timezone: "{{event.timezone}}"
    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
- date:
    field: fortinet.firewall.eventtime
    target_field: event.start
    formats:
    - UNIX_MS
    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
- date:
    field: fortinet.firewall.eventtime
    target_field: event.start
    formats:
    - UNIX
    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
- script:
    lang: painless
    source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"
    if: "ctx.fortinet?.firewall?.duration != null"
- rename:
    field: fortinet.firewall.devname
    target_field: observer.name
    ignore_missing: true
- rename:
    field: fortinet.firewall.devid
    target_field: observer.serial_number
    ignore_missing: true
- rename:
    field: fortinet.firewall.dstintf
    target_field: observer.egress.interface.name
    ignore_missing: true
    if: "ctx.observer?.egress?.interface?.name == null"
- rename:
    field: fortinet.firewall.srcintf
    target_field: observer.ingress.interface.name
    ignore_missing: true
    if: "ctx.observer?.ingress?.interface?.name == null"
- rename:
    field: fortinet.firewall.dst_int
    target_field: observer.egress.interface.name
    ignore_missing: true
- rename:
    field: fortinet.firewall.src_int
    target_field: observer.ingress.interface.name
    ignore_missing: true
- rename:
    field: fortinet.firewall.level
    target_field: log.level
    ignore_missing: true
# Handle interface-based network directionality
- set:
    field: network.direction
    value: inbound
    if: >
      ctx?._temp?.external_interfaces != null &&
      ctx?._temp?.internal_interfaces != null &&
      ctx?.observer?.ingress?.interface?.name != null &&
      ctx?.observer?.egress?.interface?.name != null &&
      ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) &&
      ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name)
- set:
    field: network.direction
    value: outbound
    if: >
      ctx?._temp?.external_interfaces != null &&
      ctx?._temp?.internal_interfaces != null &&
      ctx?.observer?.ingress?.interface?.name != null &&
      ctx?.observer?.egress?.interface?.name != null &&
      ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
      ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
- set:
    field: network.direction
    value: internal
    if: >
      ctx?._temp?.external_interfaces != null &&
      ctx?._temp?.internal_interfaces != null &&
      ctx?.observer?.ingress?.interface?.name != null &&
      ctx?.observer?.egress?.interface?.name != null &&
      ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name) &&
      ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
- set:
    field: network.direction
    value: external
    if: >
      ctx?._temp?.external_interfaces != null &&
      ctx?._temp?.internal_interfaces != null &&
      ctx?.observer?.ingress?.interface?.name != null &&
      ctx?.observer?.egress?.interface?.name != null &&
      ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
      ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name)
- set:
    field: network.direction
    value: unknown
    if: >
      ctx?._temp?.external_interfaces != null &&
      ctx?._temp?.internal_interfaces != null &&
      ctx?.observer?.egress?.interface?.name != null &&
      ctx?.observer?.ingress?.interface?.name != null &&
      (
        (
          !ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) &&
          !ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name)
        ) ||
        (
          !ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) &&
          !ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
        )
      )
- remove:
    field:
      - message
    ignore_missing: true
- pipeline:
    name: '{< IngestPipeline "event" >}'
    if: "ctx.fortinet?.firewall?.type == 'event'"
- pipeline:
    name: '{< IngestPipeline "traffic" >}'
    if: "ctx.fortinet?.firewall?.type == 'traffic'"
- pipeline:
    name: '{< IngestPipeline "utm" >}'
    if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
- rename:
    field: fortinet.firewall.reason
    target_field: event.reason
    ignore_missing: true
- rename:
    field: fortinet.firewall.msg
    target_field: message
    ignore_missing: true
- rename:
    field: fortinet.firewall.proto
    target_field: network.iana_number
    ignore_missing: true
- script:
    lang: painless
    ignore_failure: true
    if: ctx?.network?.iana_number != null
    source: |
      if (ctx?.network == null) {
          ctx.network = new HashMap();
      }
      def iana_number = ctx.network.iana_number;
      if (iana_number == '1') {
        ctx.network.transport = 'icmp';
      } else if (iana_number == '2') {
        ctx.network.transport = 'igmp';
      } else if (iana_number == '6') {
        ctx.network.transport = 'tcp';
      } else if (iana_number == '17') {
        ctx.network.transport = 'udp';
      } else if (iana_number == '58') {
        ctx.network.transport = 'ipv6-icmp';
      }
- rename:
    field: fortinet.firewall.group
    target_field: source.user.group.name
    ignore_missing: true
- uri_parts:
    field: fortinet.firewall.url
    remove_if_successful: true
    ignore_failure: true
    if: "ctx.fortinet?.firewall?.url != null"
- set:
    field: url.domain
    value: "{{fortinet.firewall.hostname}}"
    ignore_empty_value: true
    if: "ctx?.url?.domain == null"
- rename:
    field: fortinet.firewall.service
    target_field: network.protocol
    ignore_missing: true
- lowercase:
    field: network.protocol
    ignore_missing: true
- set:
    field: network.type
    value: ipv4
    if: (ctx.source?.ip != null && ctx.source?.ip.contains('.')) || (ctx.destination?.ip != null && ctx.destination?.ip.contains('.'))
- set:
    field: network.type
    value: ipv6
    if: ctx.source?.ip != null && ctx.source?.ip.contains(':') || (ctx.destination?.ip != null && ctx.destination?.ip.contains(':'))
- community_id:
    ignore_missing: true
    ignore_failure: true
- user_agent:
    field: fortinet.firewall.agent
    ignore_missing: true
- convert:
    field: fortinet.firewall.quotamax
    type: long
    ignore_missing: true
- convert:
    field: fortinet.firewall.quotaused
    type: long
    ignore_missing: true
- convert:
    field: fortinet.firewall.size
    type: long
    ignore_missing: true
- geoip:
    field: source.ip
    target_field: source.geo
    ignore_missing: true
    if: "ctx.source?.geo == null"
- geoip:
    field: destination.ip
    target_field: destination.geo
    ignore_missing: true
    if: "ctx.destination?.geo == null"
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: destination.ip
    target_field: destination.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- geoip:
    field: source.nat.ip
    target_field: source.geo
    ignore_missing: true
    if: "ctx.source?.geo == null"
- geoip:
    field: destination.nat.ip
    target_field: destination.geo
    ignore_missing: true
    if: "ctx.destination?.geo == null"
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.nat.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
    if: "ctx.source?.as == null"
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: destination.nat.ip
    target_field: destination.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
    if: "ctx.destination?.as == null"
- rename:
    field: source.as.asn
    target_field: source.as.number
    ignore_missing: true
- rename:
    field: source.as.organization_name
    target_field: source.as.organization.name
    ignore_missing: true
- rename:
    field: destination.as.asn
    target_field: destination.as.number
    ignore_missing: true
- rename:
    field: destination.as.organization_name
    target_field: destination.as.organization.name
    ignore_missing: true
- script:
    lang: painless
    source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes"
    if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null"
    ignore_failure: true
- script:
    lang: painless
    source: "ctx.network.packets = ctx.source.packets + ctx.destination.packets"
    if: "ctx?.source?.packets != null && ctx?.destination?.packets != null"
    ignore_failure: true
- append:
    field: related.ip
    value: "{{source.ip}}"
    allow_duplicates: false
    if: "ctx.source?.ip != null"
- append:
    field: related.ip
    value: "{{destination.ip}}"
    allow_duplicates: false
    if: "ctx.destination?.ip != null"
- append:
    field: related.ip
    value: "{{source.nat.ip}}"
    allow_duplicates: false
    if: "ctx.source?.nat?.ip != null"
- append:
    field: related.ip
    value: "{{destination.nat.ip}}"
    allow_duplicates: false
    if: "ctx.destination?.nat?.ip != null"
- append:
    field: related.ip
    value: "{{fortinet.firewall.ip}}"
    allow_duplicates: false
    if: "ctx.fortinet?.firewall?.ip != null"
- append:
    field: related.ip
    value: "{{fortinet.firewall.assignip}}"
    allow_duplicates: false
    if: "ctx.fortinet?.firewall?.assignip != null"
- append:
    field: related.ip
    value: "{{fortinet.firewall.tunnelip}}"
    allow_duplicates: false
    if: "ctx.fortinet?.firewall?.tunnelip != null"
- append:
    field: related.user
    value: "{{source.user.name}}"
    allow_duplicates: false
    if: "ctx.source?.user?.name != null"
- append:
    field: related.user
    value: "{{destination.user.name}}"
    allow_duplicates: false
    if: "ctx.destination?.user?.name != null"
- append:
    field: related.hosts
    value: "{{destination.address}}"
    allow_duplicates: false
    if: "ctx.destination?.address != null"
- append:
    field: related.hosts
    value: "{{source.address}}"
    allow_duplicates: false
    if: "ctx.source?.address != null"
- append:
    field: related.hosts
    value: "{{dns.question.name}}"
    allow_duplicates: false
    if: "ctx.dns?.question?.name != null"
- script:
    lang: painless
    source: |
      def dnsIPs = ctx?.dns?.resolved_ip;
      if (dnsIPs != null && dnsIPs instanceof List) {
        if (ctx?.related?.ip == null) {
          ctx.related.ip = [];
        }
        for (ip in dnsIPs) {
          if (!ctx.related.ip.contains(ip)) {
            ctx.related.ip.add(ip);
          }
        }
      }
- remove:
    field:
      - _temp
      - host
      - syslog5424_sd
      - syslog5424_pri
      - fortinet.firewall.agent
      - fortinet.firewall.date
      - fortinet.firewall.devid
      - fortinet.firewall.duration
      - fortinet.firewall.eventtime
      - fortinet.firewall.hostname
      - fortinet.firewall.time
      - fortinet.firewall.tz
      - fortinet.firewall.url
    ignore_missing: true
- script:
    lang: painless
    description: This script processor iterates over the whole document to remove fields with null values.
    source: |
      void handleMap(Map map) {
        for (def x : map.values()) {
          if (x instanceof Map) {
              handleMap(x);
          } else if (x instanceof List) {
              handleList(x);
          }
        }
        map.values().removeIf(v -> v == null);
      }
      void handleList(List list) {
        for (def x : list) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
        }
      }
      handleMap(ctx);
on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'