244 lines
7.5 KiB
JSON
244 lines
7.5 KiB
JSON
{
|
|
"description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",
|
|
"processors": [
|
|
{
|
|
"set": {
|
|
"field": "event.ingested",
|
|
"value": "{{_ingest.timestamp}}"
|
|
}
|
|
},
|
|
{
|
|
"rename": {
|
|
"field": "message",
|
|
"target_field": "event.original"
|
|
}
|
|
},
|
|
{
|
|
"grok": {
|
|
"ignore_missing": true,
|
|
"field": "event.original",
|
|
"patterns": [
|
|
"(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"(-|%{DATA:http.request.referrer})\" \"(-|%{DATA:user_agent.original})\" \"(-|%{DATA:http.request.forwarded_for})\" \"(-|%{DATA:http.request.site})\" %{NUMBER:http.request.duration}"
|
|
],
|
|
"pattern_definitions": {
|
|
"NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?",
|
|
"NGINX_NOTSEPARATOR": "[^\t ,:]+",
|
|
"NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"grok": {
|
|
"field": "nginx.access.info",
|
|
"patterns": [
|
|
"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}",
|
|
""
|
|
],
|
|
"ignore_missing": true
|
|
}
|
|
},
|
|
{
|
|
"uri_parts": {
|
|
"field": "_tmp.url_orig",
|
|
"ignore_failure": true
|
|
}
|
|
},
|
|
{
|
|
"set": {
|
|
"if": "ctx.url?.domain == null && ctx.destination?.domain != null",
|
|
"field": "url.domain",
|
|
"value": "{{destination.domain}}"
|
|
}
|
|
},
|
|
{
|
|
"remove": {
|
|
"ignore_missing": true,
|
|
"field": [
|
|
"nginx.access.info",
|
|
"_tmp.url_orig"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"split": {
|
|
"field": "nginx.access.remote_ip_list",
|
|
"separator": "\"?,?\\s+",
|
|
"ignore_missing": true
|
|
}
|
|
},
|
|
{
|
|
"split": {
|
|
"ignore_missing": true,
|
|
"field": "nginx.access.origin",
|
|
"separator": "\"?,?\\s+"
|
|
}
|
|
},
|
|
{
|
|
"set": {
|
|
"field": "source.address",
|
|
"if": "ctx.source?.address == null",
|
|
"value": ""
|
|
}
|
|
},
|
|
{
|
|
"script": {
|
|
"lang": "painless",
|
|
"source": "boolean isPrivate(def dot, def ip) {\n try {\n StringTokenizer tok = new StringTokenizer(ip, dot);\n int firstByte = Integer.parseInt(tok.nextToken());\n int secondByte = Integer.parseInt(tok.nextToken());\n if (firstByte == 10) {\n return true;\n }\n if (firstByte == 192 && secondByte == 168) {\n return true;\n }\n if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {\n return true;\n }\n if (firstByte == 127) {\n return true;\n }\n return false;\n }\n catch (Exception e) {\n return false;\n }\n} try {\n ctx.source.address = null;\n if (ctx.nginx.access.remote_ip_list == null) {\n return;\n }\n def found = false;\n for (def item : ctx.nginx.access.remote_ip_list) {\n if (!isPrivate(params.dot, item)) {\n ctx.source.address = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.source.address = ctx.nginx.access.remote_ip_list[0];\n }\n} catch (Exception e) {\n ctx.source.address = null;\n}",
|
|
"params": {
|
|
"dot": "."
|
|
},
|
|
"if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0"
|
|
}
|
|
},
|
|
{
|
|
"remove": {
|
|
"field": "source.address",
|
|
"if": "ctx.source.address == null"
|
|
}
|
|
},
|
|
{
|
|
"grok": {
|
|
"ignore_failure": true,
|
|
"field": "source.address",
|
|
"patterns": [
|
|
"^%{IP:source.ip}$"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"rename": {
|
|
"field": "@timestamp",
|
|
"target_field": "event.created"
|
|
}
|
|
},
|
|
{
|
|
"date": {
|
|
"target_field": "@timestamp",
|
|
"formats": [
|
|
"dd/MMM/yyyy:H:m:s Z"
|
|
],
|
|
"field": "nginx.access.time",
|
|
"on_failure": [
|
|
{
|
|
"append": {
|
|
"field": "error.message",
|
|
"value": "{{ _ingest.on_failure_message }}"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"remove": {
|
|
"field": "nginx.access.time"
|
|
}
|
|
},
|
|
{
|
|
"user_agent": {
|
|
"field": "user_agent.original",
|
|
"ignore_missing": true
|
|
}
|
|
},
|
|
{
|
|
"geoip": {
|
|
"field": "source.ip",
|
|
"target_field": "source.geo",
|
|
"ignore_missing": true
|
|
}
|
|
},
|
|
{
|
|
"geoip": {
|
|
"properties": [
|
|
"asn",
|
|
"organization_name"
|
|
],
|
|
"ignore_missing": true,
|
|
"database_file": "GeoLite2-ASN.mmdb",
|
|
"field": "source.ip",
|
|
"target_field": "source.as"
|
|
}
|
|
},
|
|
{
|
|
"rename": {
|
|
"ignore_missing": true,
|
|
"field": "source.as.asn",
|
|
"target_field": "source.as.number"
|
|
}
|
|
},
|
|
{
|
|
"rename": {
|
|
"field": "source.as.organization_name",
|
|
"target_field": "source.as.organization.name",
|
|
"ignore_missing": true
|
|
}
|
|
},
|
|
{
|
|
"set": {
|
|
"value": "event",
|
|
"field": "event.kind"
|
|
}
|
|
},
|
|
{
|
|
"append": {
|
|
"field": "event.category",
|
|
"value": "web"
|
|
}
|
|
},
|
|
{
|
|
"append": {
|
|
"field": "event.type",
|
|
"value": "access"
|
|
}
|
|
},
|
|
{
|
|
"set": {
|
|
"field": "event.outcome",
|
|
"value": "success",
|
|
"if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
|
|
}
|
|
},
|
|
{
|
|
"set": {
|
|
"field": "event.outcome",
|
|
"value": "failure",
|
|
"if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
|
|
}
|
|
},
|
|
{
|
|
"append": {
|
|
"value": "{{source.ip}}",
|
|
"if": "ctx?.source?.ip != null",
|
|
"field": "related.ip"
|
|
}
|
|
},
|
|
{
|
|
"append": {
|
|
"value": "{{destination.ip}}",
|
|
"if": "ctx?.destination?.ip != null",
|
|
"field": "related.ip"
|
|
}
|
|
},
|
|
{
|
|
"append": {
|
|
"value": "{{user.name}}",
|
|
"if": "ctx?.user?.name != null",
|
|
"field": "related.user"
|
|
}
|
|
},
|
|
{
|
|
"script": {
|
|
"lang": "painless",
|
|
"description": "This script processor iterates over the whole document to remove fields with null values.",
|
|
"source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"
|
|
}
|
|
}
|
|
],
|
|
"on_failure": [
|
|
{
|
|
"set": {
|
|
"field": "error.message",
|
|
"value": "{{ _ingest.on_failure_message }}"
|
|
}
|
|
}
|
|
]
|
|
} |