zeek category update and minor field updates

tools/config/arcsight-zeek.yml

@@ -15,12 +15,14 @@ logsources:
       service: syslog
   zeek-category-firewall:
     category: firewall
-    conditions:
-      deviceEventCategory: conn
+    rewrite:
+      product: zeek
+      service: conn
   zeek-category-dns:
     category: dns
-    conditions:
-      deviceEventCategory: dns
+    rewrite:
+      product: zeek
+      service: dns
   zeek-category-proxy:
     category: proxy
     rewrite:
@@ -28,8 +30,6 @@ logsources:
       service: http
   zeek-category-webserver:
     category: webserver
-    conditions:
-      deviceEventCategory: http
     rewrite:
       product: zeek
       service: http
@@ -321,7 +321,6 @@ fieldmappings:
    - destinationDnsDomain
    - destinationHost
   # All Logs Applied Mapping & Taxonomy
-  clientip: sourceAddress
   dst: destinationAddress
   dst_ip: destinationAddress
   dst_port: destinationPort
@@ -1050,4 +1049,9 @@ fieldmappings:
     - sourceAddress
   san.uri:
     - requestUrl
-    - requestUrlQuery
+    - requestUrlQuery
+  # Few other variations of names from zeek source itself
+  id_orig_h: sourceAddress
+  id_orig_p: sourcePort
+  id_resp_h: destinationAddress
+  id_resp_p: destinationPort

tools/config/ecs-zeek-corelight.yml

@@ -27,24 +27,24 @@ logsources:
       service: syslog
   zeek-category-firewall:
     category: firewall
-    conditions:
-      event.dataset: conn
+    rewrite:
+      product: zeek
+      service: conn
   zeek-category-dns:
     category: dns
-    conditions:
-      event.dataset: dns
+    rewrite:
+      product: zeek
+      service: dns
   zeek-category-proxy:
     category: proxy
     rewrite:
-        product: zeek
-        service: http
+      product: zeek
+      service: http
   zeek-category-webserver:
     category: webserver
-    conditions:
-      event.dataset: http
     rewrite:
-        product: zeek
-        service: http
+      product: zeek
+      service: http
   zeek-conn:
     product: zeek
     service: conn
@@ -397,150 +397,134 @@ fieldmappings:
   uids: log.id.uids
   uuid: log.id.uuid
   # Overlapping fields/mappings (aka: shared fields)
-  action:
-    #- smb.action
-    - '*.action'
-    #service=smb_files: smb.action
-    #service=mqtt: mqtt.action
-    #service=tunnel: tunnel.action
-  addl:
-    #- weird.addl
-    - '*.addl'
-    #service=dns: dns.addl
-    #service=weird: weird.addl
-  analyzer:
-    #- dpd.analyzer
-    - '*.analyzer'
-    #service=dpd: dpd.analyzer
-    #service=files: files.analyzer
-  arg:
-    #- ftp.arg
-    - '*.arg'
-    #service=ftp: ftp.arg
-    #service=ftp: pop3.arg
-    #service=msqyl: mysql.arg
+  #_action
+  action: '*.action'
+  mqtt_action: smb.action
+  smb_action: smb.action
+  tunnel_action: tunnel.action
+  #_addl
+  addl:  weird.addl
+  dns_addl: dns.addl
+  weird_addl: weird.addl
+  #_analyzer
+  analyzer: '*.analyzer'
+  dpd_analyzer: dpd.analyzer
+  files_analyzer: file.analyzer
+  #_arg
+  arg: '*.arg'
+  ftp_arg: ftp.arg
+  pop3_arg: pop3.arg
+  mysql_arg: mysql.arg
+  #_auth
   #auth:
     #service=rfb: rfb.auth #RFB does not exist in newer logs, so skipping to cover dns.auth
-  cipher:
-    #- kerberos.cipher
-    - '*.client'
-    #service=kerberos: kerberos.cipher
-    #service=ssl: tls.cipher
-  client:
-    #- ssh.client
-    - '*.client'
-    #service=kerberos: kerberos.client
-    #service=ssh: ssh.client
-  command:
-    #- ftp.command
-    - '*.command'
-    #service=pop3: pop3.command
-    #service=ftp: ftp.command
-    #service=irc: irc.command
-  date:
-    #- smtp.date
-    - '*.date'
-    #service=sip: sip.date
-    #service=smtp: smtp.date
-  duration:
-    - event.duration
-    #- '*.duration'
-    #service=conn: event.duration
-    #service=files: files.duration
-    #service=snmp: event.duration
-  from:
-    #- smtp.from
-    - '*.from'
-    #service=kerberos: kerberos.from
-    #service=smtp: smtp.from
-  is_orig:
-    - '*.is_orig'
-    #service=file: file.is_orig
-    #service=pop3: pop3.is_orig
-  local_orig:
-    - '*.local_orig'
-    #service=conn conn.local_orig
-    #service=files file.local_orig
-  method:
-    - http.request.method
-    #service=http: http.request.method
-    #service=sip: sip.method
-  msg:
-    - notice.msg
-    #service=notice: notice.msg
-    #service=pop3: pop3.msg
-  name:
-    - file.name
-    #- '*.name'
-    #service=smb_files: file.name
-    #service=software: software.name
-    #service=weird: weird.name
-  path:
-    - file.path
-    #- '*.path'
-    #service=smb_files: file.path
-    #service=smb_mapping: file.path
-    #service=smtp: smtp.path
-  reply_msg:
-    #- ftp.reply_msg
-    - '*.reply_msg'
-    #service=ftp: ftp.reply_msg
-    #service=radius: radius.reply_msg
-  reply_to:
-    #- smtp.reply_to
-    - '*.reply_to'
-    #service=sip: sip.reply_to
-    #service=smtp: smtp.reply_to
-  response_body_len:
-    - http.response.body.bytes
-    #service=http: http.response.body.bytes
-    #service=sip: sip.response_body_len
-  request_body_len:
-    - http.request.body.bytes
-    #service=http: http.response.body.bytes
-    #service=sip: sip.request_body_len
-  service:
-    #- kerberos.service
-    - '*.service'
-    #service=kerberos: kerberos.service
-    #service=smb_mapping: smb.service
-  status:
-    #- socks.status
-    - '*.status'
-    #service=pop3: pop3.status
-    #service=mqtt: mqtt.status
-    #service=socks: socks.status
-  status_code:
-    - 'http.response.status_code'
-    #service=http: http.response.status_code
-    #service=sip: sip.status_code
-  status_msg:
-    - http.status_msg
-    #- '*.status_msg'
-    #service=http: http.status_msg
-    #service=sip: sip.status_msg
-  subject:
-    #- smtp.subject
-    - '*.subject'
-    #service=known_certs: known_certs.subject
-    #service=sip: sip.subject
-    #service=smtp: smtp.subject
-    #service=ssl: tls.subject
-  trans_depth:
-    #- http.trans_depth
-    - '*.trans_depth'
-    #service=http: http.trans_depth
-    #service=sip: sip.trans_depth
-    #service=smtp: smtp.trans_depth
-  version:
-    #- tls.version
-    - '*.version'
-    #service=gquic: gquic.version
-    #service=ntp: ntp.version
-    #service=socks: socks.version
-    #service=snmp: snmp.version
-    #service=ssh: ssh.version
-    #service=tls: tls.version
+  dns_auth: dns.auth
+  rfb_auth: rfb.auth
+  #_cipher
+  cipher: tls.cipher
+  kerberos_cipher: kerberos.cipher
+  tls_cipher: tls.cipher
+  #_client
+  client: '*.client'
+  kerberos_client: kerberos.client
+  ssh_client: ssh.client
+  #_command
+  command: '*.command'
+  ftp_command: ftp.command
+  irc_command: ssh.client
+  pop3_command: pop3.command
+  #_date
+  date: '*.date'
+  sip_date: sip.date
+  smtp_date: smtp.date
+  #_duration
+  duration: event.duration
+  conn_duration: event.duration
+  files_duration: files.duration
+  snmp_duration: event.duration
+  #_from
+  from: '*.from'
+  kerberos_from: kerberos.from
+  smtp_from: smtp.from
+  #_is_orig
+  is_orig: '*.is_orig'
+  is_orig_file: file.is_orig
+  is_orig_pop3: pop3.is_orig
+  #_local_orig
+  local_orig: '*.local_orig'
+  conn_local_orig: conn.local_orig
+  files_local_orig: file.local_orig
+  #_method
+  method: http.request.method 
+  http_method: http.request.method
+  sip_method: sip.method
+  #_msg
+  msg: notice.msg
+  notice_msg: notice.msg
+  pop3_msg: pop3.msg
+  #_name
+  name: file.name
+  smb_files_name: file.name
+  software_name: software.name
+  weird_name: weird.name
+  #_path
+  path: file.path
+  smb_files_path: file.path
+  smb_mapping_path: file.path
+  smtp_path: smtp.path
+  #_reply_msg
+  reply_msg: '*.reply_msg'
+  ftp_reply_msg: ftp.reply_msg
+  radius_reply_msg: radius.reply_msg
+  #_reply_to
+  reply_to: '*.reply_to'
+  sip_reply_to: sip.reply_to
+  smtp_reply_to: smtp.reply_to
+  #_response_body_len
+  response_body_len: http.response.body.bytes
+  http_response_body_len: http.response.body.bytes
+  sip_response_body_len: sip.response_body_len
+  #_request_body_len
+  request_body_len: http.request.body.bytes
+  http_request_body_len: http.response.body.bytes
+  sip_request_body_len: sip.response_body_len
+  #_service
+  service: '*.service'
+  kerberos_service: kerberos.service
+  smb_mapping_kerberos: smb.service
+  #_status
+  status: '*.status'
+  mqtt_status: mqtt.status
+  pop3_status: pop3.status
+  socks_status: socks.status
+  #_status_code
+  status_code: 'http.response.status_code'
+  http_status_code: http.response.status_code
+  sip_status_code: sip.status_code
+  #_status_msg
+  status_msg: http.status_msg
+  http_status_msg: http.status_msg
+  sip_status_msg: sip.status_msg
+  #_subject
+  subject: tls.subject
+  known_certs_subject: known_certs.subject
+  sip_subject: sip.subject
+  smtp_subject: smtp.subject
+  ssl_subject: tls.subject
+  #_trans_depth
+  trans_depth: '*.trans_depth'
+  http_trans_depth: http.trans_depth
+  sip_trans_depth: sip.trans_depth
+  smtp_trans_depth: smtp.trans_depth
+  #_version
+  version: '*.version'
+  gquic_version: gquic.version
+  http_version: http.version
+  ntp_version: ntp.version
+  socks_version: socks.version
+  snmp_version: snmp.version
+  ssh_version: ssh.version
+  tls_version: tls.version
   # Conn and Conn Long
   cache_add_rx_ev: conn.cache_add_rx_ev
   cache_add_rx_mpg: conn.cache_add_rx_mpg
@@ -690,6 +674,7 @@ fieldmappings:
   uri_vars: http.uri_vars
   #user_agent: user_agent.original
   #username: source.user.name
+  #version: http.version
   # Intel
   file_mime_type: file.mime_type
   file_desc: intel.file_desc
@@ -1063,10 +1048,12 @@ fieldmappings:
   san.email: x509.san.email
   san.ip: x509.san.ip
   san.uri: x509.san.url
-  # Temporary one off rule name's people have written
-  agent.version: version
-  c-cookie: http.cookie_vars
-  c-ip: source.ip
+  # Few other variations of names from zeek source itself
+  id_orig_h: source.ip
+  id_orig_p: source.port
+  id_resp_h: destination.ip
+  id_resp_p: destination.port
+  # Temporary one off rule name fields
   cs-uri: url.original
   clientip: source.ip
   clientIP: source.io

tools/config/ecs-zeek-elastic-beats-implementation.yml

@@ -13,8 +13,6 @@ logsources:
   zeek:
     product: zeek
     index: 'filebeat*'
-      #'*ecs-corelight*'
-      #'*ecs-zeek-*
   zeek-category-accounting:
     category: accounting
     rewrite:
@@ -22,12 +20,14 @@ logsources:
       service: syslog
   zeek-category-firewall:
     category: firewall
-    conditions:
-      event.dataset: zeek.connection
+    rewrite:
+      product: zeek
+      service: conn
   zeek-category-dns:
     category: dns
-    conditions:
-      event.dataset: zeek.dns
+    rewrite:
+      product: zeek
+      service: dns
   zeek-category-proxy:
     category: proxy
     rewrite:
@@ -35,8 +35,6 @@ logsources:
         service: http
   zeek-category-webserver:
     category: webserver
-    conditions:
-      event.dataset: zeek.http
     rewrite:
         product: zeek
         service: http
@@ -356,135 +354,84 @@ fieldmappings:
   #user_agent: user_agent.original
   #vlan: network.vlan.id # Not implemented by Elastic (Beats) yet
   # Overlapping fields/mappings (aka: shared fields)
-  action:
-    - 'zeek.smb_files.action'
-    #service=tunnel: zeek.tunnel.action
-    #service=smb_files: zeek.smb_files.action
-  addl:
-    - 'zeek.weird.additional_info'
-    #service=dns: zeek.dns.addl
-    #service=weird: zeek.weird.additional_info
-  arg:
-    - 'zeek.*.arg'
-  auth:
-    - 'zeek.*.auth*'
-    #service=dns: zeek.dns.auth
-    #service=rfb: zeek.rfb.auth.success
-  cipher:
-    - 'zeek.*.cipher'
-    #service=kerberos: zeek.kerberos.cipher
-    #service=ssl: zeek.ssl.cipher
-  client:
-    - 'zeek.*.client*'
-    #service=kerberos: zeek.kerberos.cert.client.value
-    #service=ssh: zeek.ssh.client
-  command:
-    - 'zeek.*.command'
-    #service=ftp: zeek.ftp.command
-    #service=irc: zeek.irc.command
-  date:
-    - 'zeek.*.date'
-    #service=smtp: zeek.smtp.date
-    #service=sip: zeek.sip.date
-  duration:
-    #- event.duration
-    - '*.duration'
-    #service=conn: event.duration
-    #service=files: zeek.files.duration
-    #service=snmp: zeek.snmp.duration
-  from:
-    - 'zeek.*.from'
-    #service=smtp: zeek.smtp.from
-    #service=kerberos: zeek.kerberos.valid.from
-  is_orig:
-    - 'zeek.*.is_orig'
-  local_orig:
-    - 'zeek.*.local_orig'
-  method:
-    - http.request.method
-    #service=http: http.request.method
-    #service=sip: zeek.sip.sequence.method
-  name:
-    - 'zeek.smb_files.name'
-      #service=weird: zeek.weird.name
-      #service=smb_files: zeek.smb_files.name
-  path:
-    - 'zeek.*.path'
-    #service=smb_mapping: zeek.smb_mapping.path
-    #service=smb_files: zeek.smb_files.path
-    #service=smtp: zeek.smtp.path
-  password:
-    - 'zeek.*.password'
-    #service=ftp: zeek.ftp.password
-    #service=http: zeek.http.password
-    #service=socks: zeek.socks.password
-  reply_msg:
-    - 'zeek.*.reply*msg'
-    #service=ftp: zeek.ftp.reply.msg
-    #service=radius: zeek.radius.reply_msg
-  response_body_len:
-    - http.response.body.bytes
-    #service=http: http.response.body.bytes
-    #service=sip: zeek.sip.response_body_len
-  request_body_len:
-    - http.request.body.bytes
-    #service=http: http.response.body.bytes
-    #service=sip: zeek.sip.request_body_len
-  rtt:
-    #- event.duration
-    - 'zeek.*.rtt'
-    #service=dns: zeek.dns.rtt
-    #service=dce_rpc: zeek.dce_rpc.rtt
-  status_code:
-    - 'http.response.status_code'
-    #service=http: http.response.status_code
-    #service=sip: zeek.sip.status_code
-  status_msg:
-    - 'zeek.*status*msg'
-    #service=http: zeek.http.status_msg
-    #service=sip: zeek.sip.status.msg
+  action: 'zeek.smb_files.action'
+  mqtt_action: smb.action
+  smb_action: smb.action
+  tunnel_action: tunnel.action
+  addl: 'zeek.weird.additional_info'
+  dns_addl: zeek.dns.addl
+  weird_addl: zeek.weird.additional_info
+  arg: 'zeek.*.arg'
+  ftp_arg: zeek.ftp.arg
+  mysql_arg: zeek.mysql.arg
+  pop3_arg: zeek.pop3.arg
+  auth: 'zeek.*.auth*'
+  cipher: 'zeek.*.cipher'
+  kerberos_cipher: zeek.kerberos.cipher
+  ssl_cipher: zeek.ssl.cipher
+  tls_cipher: zeek.ssl.cipher
+  client: 'zeek.*.client*'
+  command: 'zeek.*.command'
+  ftp_command: zeek.irc.command
+  irc_command: zeek.ftp.command
+  pop3_command: zeek.pop3.command
+  date: 'zeek.*.date'
+  duration: event.duration
+  from: 'zeek.*.from'
+  kerberos_from: zeek.smtp.from
+  smtp_from: zeek.kerberos.valid.from
+  is_orig: 'zeek.*.is_orig'
+  local_orig: 'zeek.*.local_orig'
+  method: http.request.method
+  http_method: http.request.method
+  sip_method: zeek.sip.sequence.method
+  name: 'zeek.smb_files.name'
+  smb_files_name: zeek.smb_files.name
+  software_name: zeek.software.name
+  weird_name: zeek.weird.name
+  path: 'zeek.*.path'
+  smb_mapping_path: zeek.smb_mapping.path
+  smb_files_path: zeek.smb_files.path
+  smtp_files_path: zeek.smtp.path
+  password: 'zeek.*.password'
+  reply_msg: 'zeek.*.reply*msg'
+  reply_to: 'zeek.*.reply_to'
+  response_body_len: http.response.body.bytes
+  request_body_len: http.request.body.bytes
+  rtt: event.duration
+  status_code: 'http.response.status_code'
+  status_msg: 'zeek.*status*msg'
+  #_service:
+  service: 'zeek.*.service'
+  kerberos_service: zeek.kerberos.service
+  smb_mapping_kerberos: zeek.smb_mapping.service
+  #_subject:
   subject:
     - 'zeek.*.subject'
-    #service=sip: zeek.sip.subject
-    #service=ssl: zeek.ssl.subject
-  service:
-    - 'zeek.*.service'
-    #service=kerberos: zeek.kerberos.service
-    #service=smb_mapping: zeek.smb_mapping.service
-    - 'zeek.*.reply_to'
-    #service=sip: zeek.sip.reply_to
-    #service=smtp: zeek.smtp.reply_to
-  trans_depth:
-    - 'zeek.*.trans*depth'
-    #service=smtp: zeek.smtp.transaction_depth
-    #service=http: zeek.http.trans_depth
-    #service=sip: zeek.sip.transaction_depth
-  username:
-    - 'zeek.*.username'
-    #service=http: url.username
-    #service=notice: zeek.notice.username
-    #service=pop3: zeek.pop3.username
-    #service=radius: zeek.radius.username
-  uri:
-    - 'url.original'
-    #service=http: url.original
-    #service=sip: zeek.sip.uri
-  user:
-    - 'zeek.*user*'
-    #service=ftp: zeek.ftp.user.name
-    #service=irc: zeek.irc.user.name
+  known_certs_subject: zeek.known_certs.subject
+  sip_subject: zeek.sip.subject
+  smtp_subject: zeek.smtp.subject
+  ssl_subject: zeek.ssl.subject  
+  trans_depth: 'zeek.*.trans*depth'
+  username: 'zeek.*.username'
+  uri: 'url.original'
+  user: 'zeek.*user*'
+  #_user_agent
   user_agent:
     - 'zeek.*user_agent*'
-    #service=http: user_agent.original
-    #service=guic: user_agent
-    #service=sip: zeek.sip.user_agent
-    #service=smtp: zeek.smtp.user_agent
-  version:
-    - 'zeek.*.version'
-    #service=snmp: zeek.snmp.version
-    #service=socks: zeek.socks.version
-    #service=ssh: zeek.ssh.version
-    #service=ssl: zeek.ssl.version
+  http_user_agent: user_agent.original
+  gquic_user_agent: zeek.gquic.user_agent
+  sip_user_agent: zeek.sip.user_agent
+  smtp_user_agent: zeek.smtp.user_agent
+  #_version
+  version: 'zeek.*.version'
+  gquic_version: zeek.gquic.version
+  http_version: http.version
+  ntp_version: zeek.ntp.version
+  socks_version: zeek.socks.version
+  snmp_version: zeek.snmp.version
+  ssh_version: zeek.ssh.version
+  tls_version: zeek.ssl.version
   # DNS matching Taxonomy & DNS Category
   answer: dns.answers.name
   question_length: labels.dns.query_length
@@ -660,6 +607,7 @@ fieldmappings:
   uri_vars: zeek.http.uri_vars
   #user_agent: user_agent.original
   #username: source.user.name
+  #version: http.version
   # Intel
   file_mime_type: zeek.intel.mime_type
   file_desc: zeek.intel.file_desc

tools/config/logstash-zeek-default-json.yml

@@ -19,12 +19,14 @@ logsources:
       service: syslog
   zeek-category-firewall:
     category: firewall
-    conditions:
-      '@stream': conn
+    rewrite:
+      product: zeek
+      service: conn
   zeek-category-dns:
     category: dns
-    conditions:
-      '@stream': dns
+    rewrite:
+      product: zeek
+      service: dns
   zeek-category-proxy:
     category: proxy
     rewrite:
@@ -32,8 +34,6 @@ logsources:
         service: http
   zeek-category-webserver:
     category: webserver
-    conditions:
-      '@stream': http
     rewrite:
         product: zeek
         service: http
@@ -346,7 +346,12 @@ fieldmappings:
   cs-method: method
   cs-referrer: referrer
   cs-version: version
-  # Temporary one off rule name's people have written
+  # Few other variations of names from zeek source itself
+  id_orig_h: id.orig_h
+  id_orig_p: id.orig_p
+  id_resp_h: id.resp_h
+  id_resp_p: id.resp_p
+  # Temporary one off rule name fields
   agent.version: version
   c-cookie: cookie
   c-ip: id.orig_h

tools/config/splunk-zeek.yml

@@ -12,12 +12,14 @@ logsources:
       service: syslog
   zeek-category-firewall:
     category: firewall
-    conditions:
-      sourcetype: 'bro:conn:json'
+    rewrite:
+      product: zeek
+      service: conn
   zeek-category-dns:
     category: dns
-    conditions:
-      sourcetype: 'bro:dns:json'
+    rewrite:
+      product: zeek
+      service: dns
   zeek-category-proxy:
     category: proxy
     rewrite:
@@ -25,16 +27,15 @@ logsources:
       service: http
   zeek-category-webserver:
     category: webserver
-    conditions:
-      sourcetype: 'bro:http:json'
     rewrite:
       product: zeek
       service: http
   zeek-conn:
     product: zeek
     service: conn
-    conditions:
-      sourcetype: 'bro:conn:json'
+    rewrite:
+      product: zeek
+      service: conn
   zeek-conn_long:
     product: zeek
     service: conn_long
@@ -337,4 +338,9 @@ fieldmappings:
   cs-host: host
   cs-method: method
   cs-referrer: referrer
-  cs-version: version
+  cs-version: version
+  # Few other variations of names from zeek source itself
+  id_orig_h: id.orig_h
+  id_orig_p: id.orig_p
+  id_resp_h: id.resp_h
+  id_resp_p: id.resp_p