File: //usr/share/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
---
description: Pipeline for AWS CloudTrail Logs
processors:
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
- set:
field: event.created
value: '{{@timestamp}}'
- rename:
field: "message"
target_field: "event.original"
- json:
field: "event.original"
target_field: "json"
- date:
field: "json.eventTime"
target_field: "@timestamp"
ignore_failure: true
formats:
- ISO8601
- rename:
field: "json.eventVersion"
target_field: "aws.cloudtrail.event_version"
ignore_failure: true
- rename:
field: "json.userIdentity.type"
target_field: "aws.cloudtrail.user_identity.type"
ignore_failure: true
- append:
field: related.user
value: '{{json.userIdentity.userName}}'
allow_duplicates: false
if: 'ctx.json?.userIdentity?.userName != null'
- rename:
field: "json.userIdentity.userName"
target_field: "user.name"
ignore_failure: true
- rename:
field: "json.userIdentity.principalId"
target_field: "user.id"
ignore_failure: true
- rename:
field: "json.userIdentity.arn"
target_field: "aws.cloudtrail.user_identity.arn"
ignore_failure: true
- rename:
field: "json.userIdentity.accountId"
target_field: "cloud.account.id"
ignore_failure: true
- rename:
field: "json.userIdentity.accessKeyId"
target_field: "aws.cloudtrail.user_identity.access_key_id"
ignore_failure: true
- rename:
field: "json.userIdentity.sessionContext.attributes.mfaAuthenticated"
target_field: "aws.cloudtrail.user_identity.session_context.mfa_authenticated"
ignore_failure: true
- date:
field: "json.userIdentity.sessionContext.attributes.creationDate"
target_field: "aws.cloudtrail.user_identity.session_context.creation_date"
ignore_failure: true
formats:
- ISO8601
- rename:
field: "json.userIdentity.sessionContext.sessionIssuer.type"
target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type"
ignore_failure: true
# userIdentity.sessionIssuer.userName is only set with assumed roles.
- rename:
field: "json.userIdentity.sessionContext.sessionIssuer.userName"
target_field: "user.name"
ignore_failure: true
- rename:
field: "json.userIdentity.sessionContext.sessionIssuer.principalId"
target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id"
ignore_failure: true
- rename:
field: "json.userIdentity.sessionContext.sessionIssuer.arn"
target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.arn"
ignore_failure: true
- rename:
field: "json.userIdentity.sessionContext.sessionIssuer.accountId"
target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.account_id"
ignore_failure: true
- rename:
field: "json.userIdentity.invokedBy"
target_field: "aws.cloudtrail.user_identity.invoked_by"
ignore_failure: true
- rename:
field: "json.eventSource"
target_field: "event.provider"
ignore_failure: true
- set:
field: "event.action"
value: "{{json.eventName}}"
ignore_failure: true
ignore_empty_value: true
- rename:
field: "json.eventCategory"
target_field: "aws.cloudtrail.event_category"
ignore_failure: true
- set:
field: "cloud.region"
copy_from: "json.awsRegion"
ignore_empty_value: true
- rename:
field: "json.sourceIPAddress"
target_field: "source.address"
ignore_failure: true
- grok:
field: source.address
ignore_failure: true
patterns:
- ^%{IP:source.ip}$
- geoip:
field: "source.ip"
target_field: "source.geo"
ignore_failure: true
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: source.ip
target_field: source.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: source.as.asn
target_field: source.as.number
ignore_missing: true
- rename:
field: source.as.organization_name
target_field: source.as.organization.name
ignore_missing: true
- user_agent:
field: "json.userAgent"
target_field: "user_agent"
on_failure:
- rename:
field: "json.userAgent"
target_field: "user_agent.original"
ignore_failure: true
- rename:
field: "json.errorCode"
target_field: "aws.cloudtrail.error_code"
ignore_failure: true
- rename:
field: "json.errorMessage"
target_field: "aws.cloudtrail.error_message"
ignore_failure: true
- script:
lang: painless
source: |
if (ctx.aws.cloudtrail?.flattened == null) {
Map map = new HashMap();
ctx.aws.cloudtrail.put("flattened", map);
}
if (ctx.json?.requestParameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
}
}
if (ctx.json?.responseElements != null) {
ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
}
}
if (ctx.json?.additionalEventData != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
}
}
if (ctx.json?.serviceEventDetails != null) {
ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);
}
}
ignore_failure: true
- rename:
field: "json.requestID"
target_field: "aws.cloudtrail.request_id"
ignore_failure: true
- rename:
field: "json.eventID"
target_field: event.id
ignore_failure: true
- rename:
field: "json.eventType"
target_field: "aws.cloudtrail.event_type"
ignore_failure: true
- rename:
field: "json.apiVersion"
target_field: "aws.cloudtrail.api_version"
ignore_failure: true
- rename:
field: "json.managementEvent"
target_field: "aws.cloudtrail.management_event"
ignore_failure: true
- rename:
field: "json.readOnly"
target_field: "aws.cloudtrail.read_only"
ignore_failure: true
- rename:
field: "json.resources.ARN"
target_field: "aws.cloudtrail.resources.arn"
ignore_failure: true
- rename:
field: "json.resources.accountId"
target_field: "aws.cloudtrail.resources.account_id"
ignore_failure: true
- rename:
field: "json.resources.type"
target_field: "aws.cloudtrail.resources.type"
ignore_failure: true
- rename:
field: "json.recipientAccountId"
target_field: "aws.cloudtrail.recipient_account_id"
ignore_failure: true
- rename:
field: "json.sharedEventId"
target_field: "aws.cloudtrail.shared_event_id"
ignore_failure: true
- rename:
field: "json.vpcEndpointId"
target_field: "aws.cloudtrail.vpc_endpoint_id"
ignore_failure: true
- append:
field: related.user
value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
allow_duplicates: false
if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null'
- append:
field: related.user
value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
allow_duplicates: false
if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null'
- script:
lang: painless
ignore_failure: true
source: >-
if (ctx.json?.eventName != 'ConsoleLogin') {
return;
}
Map aed_map = new HashMap();
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') {
aed_map.put("mobile_version", false);
} else {
aed_map.put("mobile_version", true);
}
}
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) {
aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo);
}
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') {
aed_map.put("mfa_used", false);
} else {
aed_map.put("mfa_used", true);
}
}
if (aed_map.size() > 0) {
Map cl_map = new HashMap();
cl_map.put("additional_eventdata", aed_map);
ctx.aws.cloudtrail.put("console_login", cl_map);
}
- script:
lang: painless
ignore_failure: true
params:
AddUserToGroup:
category:
- iam
type:
- group
- change
AssumeRole:
category:
- authentication
type:
- info
AttachGroupPolicy:
category:
- iam
type:
- group
- change
AttachUserPolicy:
category:
- iam
type:
- user
- change
ChangePassword:
category:
- iam
type:
- user
- change
ConsoleLogin:
category:
- authentication
type:
- info
CreateAccessKey:
category:
- iam
type:
- user
- change
CreateBucket:
category:
- file
type:
- creation
CreateGroup:
category:
- iam
type:
- group
- creation
CreateKeyPair:
category:
- iam
type:
- admin
- creation
CreateUser:
category:
- iam
type:
- user
- creation
CreateVirtualMFADevice:
category:
- iam
type:
- user
- change
DeactivateMFADevice:
category:
- iam
type:
- user
- change
DeleteAccessKey:
category:
- iam
type:
- user
- change
DeleteBucket:
category:
- file
type:
- deletion
DeleteGroup:
category:
- iam
type:
- group
- deletion
DeleteGroupPolicy:
category:
- iam
type:
- group
- change
DeleteSSHPublicKey:
category:
- iam
type:
- user
- change
DeleteUser:
category:
- iam
type:
- user
- deletion
DeleteUserPermissionsBoundary:
category:
- iam
type:
- user
- change
DeleteUserPolicy:
category:
- iam
type:
- user
- change
DeleteVirtualMFADevice:
category:
- iam
type:
- user
- change
DetachGroupPolicy:
category:
- iam
type:
- group
- change
DetachUserPolicy:
category:
- iam
type:
- user
- change
EnableMFADevice:
category:
- iam
type:
- user
- change
GetGroup:
category:
- iam
type:
- group
- info
GetGroupPolicy:
category:
- iam
type:
- group
- info
GetUser:
category:
- iam
type:
- user
- info
GetUserPolicy:
category:
- iam
type:
- user
- info
ListAttachedGroupPolicies:
category:
- iam
type:
- group
- info
ListAttachedUserPolicies:
category:
- iam
type:
- user
- info
ListGroupPolicies:
category:
- iam
type:
- group
- info
ListGroups:
category:
- iam
type:
- group
- info
ListGroupsForUser:
category:
- iam
type:
- user
- info
ListUserPolicies:
category:
- iam
type:
- user
- info
ListUsers:
category:
- iam
type:
- user
- info
ListUserTags:
category:
- iam
type:
- user
- info
PutGroupPolicy:
category:
- iam
type:
- group
- change
PutUserPermissionsBoundary:
category:
- iam
type:
- user
- change
PutUserPolicy:
category:
- iam
type:
- user
- change
RemoveUserFromGroup:
category:
- iam
type:
- group
- change
SetDefaultPolicyVersion:
category:
- iam
type:
- admin
- change
SetSecurityTokenServicePreferences:
category:
- iam
type:
- admin
- change
TagUser:
category:
- iam
type:
- user
- change
UntagUser:
category:
- iam
type:
- user
- change
UpdateAccessKey:
category:
- iam
type:
- user
- change
UpdateAccountPasswordPolicy:
category:
- iam
type:
- admin
- change
UpdateGroup:
category:
- iam
type:
- group
- change
UpdateLoginProfile:
category:
- iam
type:
- user
- change
UpdateRole:
category:
- iam
type:
- admin
- change
UpdateSSHPublicKey:
category:
- iam
type:
- user
- change
UpdateUser:
category:
- iam
type:
- user
- change
source: >-
ctx.event.kind = 'event';
ctx.event.type = 'info';
if (ctx.aws.cloudtrail.error_code != null || ctx.aws.cloudtrail.error_message != null) {
ctx.event.outcome = 'failure'
} else {
ctx.event.outcome = 'success'
}
if (ctx?.event?.action == null) {
return;
}
if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) {
ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin);
}
def hm = new HashMap(params.get(ctx.event.action));
hm.forEach((k, v) -> ctx.event[k] = v);
- rename:
field: "json.awsAccountId"
target_field: "cloud.account.id"
ignore_failure: true
- rename:
field: "json.digestS3Object"
target_field: "file.path"
ignore_failure: true
- rename:
field: "json.previousDigestSignature"
target_field: "file.hash.sha256"
if: >-
ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256'
- append:
field: "related.hash"
value: "{{json.previousDigestSignature}}"
if: "ctx?.json?.previousDigestSignature != null"
- rename:
field: "json.logFiles"
target_field: "aws.cloudtrail.digest.log_files"
ignore_failure: true
- date:
field: "json.digestStartTime"
target_field: "aws.cloudtrail.digest.start_time"
ignore_failure: true
formats:
- ISO8601
- date:
field: "json.digestEndTime"
target_field: "@timestamp"
ignore_failure: true
formats:
- ISO8601
- date:
field: "json.digestEndTime"
target_field: "aws.cloudtrail.digest.end_time"
ignore_failure: true
formats:
- ISO8601
- rename:
field: "json.digestS3Bucket"
target_field: "aws.cloudtrail.digest.s3_bucket"
ignore_failure: true
- date:
field: "json.newestEventTime"
target_field: "aws.cloudtrail.digest.newest_event_time"
ignore_failure: true
formats:
- ISO8601
- date:
field: "json.oldestEventTime"
target_field: "aws.cloudtrail.digest.oldest_event_time"
ignore_failure: true
formats:
- ISO8601
- rename:
field: "json.previousDigestS3Bucket"
target_field: "aws.cloudtrail.digest.previous_s3_bucket"
ignore_failure: true
- rename:
field: "json.previousDigestHashAlgorithm"
target_field: "aws.cloudtrail.digest.previous_hash_algorithm"
ignore_failure: true
- rename:
field: "json.publicKeyFingerprint"
target_field: "aws.cloudtrail.digest.public_key_fingerprint"
ignore_failure: true
- rename:
field: "json.digestSignatureAlgorithm"
target_field: "aws.cloudtrail.digest.signature_algorithm"
ignore_failure: true
- rename:
field: "json.insightDetails"
target_field: "aws.cloudtrail.insight_details"
ignore_failure: true
- set:
field: group.id
value: '{{aws.cloudtrail.flattened.response_elements.group.groupId}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.target.id
value: '{{aws.cloudtrail.flattened.response_elements.user.userId}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.changes.name
value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: group.name
value: '{{aws.cloudtrail.flattened.request_parameters.groupName}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.target.name
value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
ignore_empty_value: true
ignore_failure: true
- remove:
field:
- "json"
ignore_missing: true
on_failure:
- set:
field: "error.message"
value: "{{ _ingest.on_failure_message }}"