metron-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From justinleet <...@git.apache.org>
Subject [GitHub] metron issue #734: METRON-1158: Build backend for grouping alerts into meta ...
Date Wed, 06 Sep 2017 19:36:51 GMT
Github user justinleet commented on the issue:

    https://github.com/apache/metron/pull/734
  
    ## Create Meta Alert
    
    The first thing to do is to build and deploy full dev with this PR.
    
    Afterwards, we're going to set up a bit of base data.
    Retrieve the current list of indices so we know where to put our data
    ```
    curl 'node1:9200/_cat/indices?v'
    health status index                     pri rep docs.count docs.deleted store.size pri.store.size
    green  open   snort_index_2017.09.06.14   1   0        130            0    180.9kb   
    180.9kb
    green  open   bro_index_2017.09.06.14     1   0        160            0    564.3kb   
    564.3kb
    green  open   .kibana                     1   0         52            0     71.2kb   
     71.2kb
    green  open   metaalerts                  1   0          6            0     62.3kb   
     62.3kb
    ```
    In this case, we care about `snort_index_2017.09.06.14` and `metaalerts`.  To make our
lives easier, we'll add a couple of stripped down messages to the our snort index (Make sure
to sub in the correct index name):
    ```
    curl -XPUT 'node1:9200/snort_index_2017.09.06.14/snort_doc/snort_test_1?pretty' -H 'Content-Type:
application/json' -d'
    {
      "msg": "snort test alert",
      "ip_dst_port": "8080",
      "ethsrc": "0A:00:27:00:00:00",
      "protocol": "TCP",
      "source:type": "snort",
      "ip_dst_addr": "192.168.66.121",
      "ip_src_addr": "192.168.66.1",
      "threat:triage:rules:0:score": 10,
      "timestamp": 1504708744000,
      "threat:triage:rules:0:reason": null,
      "threat:triage:score": 10,
      "is_alert": "true",
      "ip_src_port": "50187",
      "guid": "snort_test_1"
    }
    '
    ```
    
    ```
    curl -XPUT 'node1:9200/snort_index_2017.09.06.14/snort_doc/snort_test_2?pretty' -H 'Content-Type:
application/json' -d'
    {
      "msg": "snort test alert 2",
      "ip_dst_port": "8080",
      "ethsrc": "0A:00:27:00:00:00",
      "protocol": "TCP",
      "source:type": "snort",
      "ip_dst_addr": "192.168.66.121",
      "ip_src_addr": "192.168.66.1",
      "threat:triage:rules:0:score": 10,
      "timestamp": 1504708744000,
      "threat:triage:rules:0:reason": null,
      "threat:triage:score": 10,
      "is_alert": "true",
      "ip_src_port": "50187",
      "guid": "snort_test_2"
    }
    '
    ```
    
    At this point, we'll group these alerts together.  In Ambari, go to Metron -> Quick
Links -> Swagger UI and go to the "Meta Alert Controller".
    Click on the create endpoint, and we'll want to send this request (but with the appropriate
index subbed in):
    ```
    {
      "groups": [
        "group_one",
        "group_two"
      ],
      "guidToIndices": {
        "snort_test_1":"snort_index_2017.09.06.14",
        "snort_test_2":"snort_index_2017.09.06.14"
      }
    }
    ```
    
    Validate that the metaalert was created and looks good:
    ```
    curl 'node1:9200/m*/_search?pretty'
    {
      "took" : 3,
      "timed_out" : false,
      "_shards" : {
        "total" : 1,
        "successful" : 1,
        "failed" : 0
      },
      "hits" : {
        "total" : 1,
        "max_score" : 1.0,
        "hits" : [ {
          "_index" : "metaalerts",
          "_type" : "metaalert_doc",
          "_id" : "d414a9f2-7a03-4e47-9108-16b6fc800dd3",
          "_score" : 1.0,
          "_timestamp" : 1504725383439,
          "_source" : {
            "average" : 10.0,
            "min" : 10.0,
            "median" : 10.0,
            "alert" : [ {
              "msg" : "snort test alert",
              "threat:triage:rules:0:reason" : null,
              "ip_dst_port" : "8080",
              "ethsrc" : "0A:00:27:00:00:00",
              "threat:triage:score" : 10,
              "is_alert" : "true",
              "protocol" : "TCP",
              "source:type" : "snort",
              "ip_dst_addr" : "192.168.66.121",
              "ip_src_port" : "50187",
              "guid" : "snort_test_1",
              "ip_src_addr" : "192.168.66.1",
              "threat:triage:rules:0:score" : 10,
              "timestamp" : 1504708744000
            }, {
              "msg" : "snort test alert 2",
              "threat:triage:rules:0:reason" : null,
              "ip_dst_port" : "8080",
              "ethsrc" : "0A:00:27:00:00:00",
              "threat:triage:score" : 10,
              "is_alert" : "true",
              "protocol" : "TCP",
              "source:type" : "snort",
              "ip_dst_addr" : "192.168.66.121",
              "ip_src_port" : "50187",
              "guid" : "snort_test_2",
              "ip_src_addr" : "192.168.66.1",
              "threat:triage:rules:0:score" : 10,
              "timestamp" : 1504708744000
            } ],
            "max" : 10.0,
            "threat:triage:score" : 20.0,
            "count" : 2,
            "guid" : "d414a9f2-7a03-4e47-9108-16b6fc800dd3",
            "groups" : [ "group_one", "group_two" ],
            "sum" : 20.0,
            "status" : "active"
          }
        } ]
      }
    }
    ```
    
    Note that both sub alerts are present, the various counts are filled in, a GUID has been
given specifically to this meta alert, etc.
    
    ## Searching
    
    Searching from the REST API works mostly as expected.  After the above data has been created,
use the search endpoint to run this query:
    ```
    {
      "from": 0,
      "size": 2,
      "indices": [
        "*"
      ],
      "query": "guid=snort_test_2",
      "sort": [
        {
          "field": "_timestamp",
          "sortOrder": "ASC"
        }
      ]
    }
    ```
    The result should look similar to the following, and have the messages we'd expect (one
from snort and one from metaalerts):
    ```
    
    {
      "total": 2,
      "results": [
        {
          "id": "snort_test_2",
          "source": {
            "msg": "snort test alert 2",
            "threat:triage:rules:0:reason": null,
            "ip_dst_port": "8080",
            "ethsrc": "0A:00:27:00:00:00",
            "threat:triage:score": 10,
            "is_alert": "true",
            "protocol": "TCP",
            "source:type": "snort",
            "ip_dst_addr": "192.168.66.121",
            "ip_src_port": "50187",
            "guid": "snort_test_2",
            "ip_src_addr": "192.168.66.1",
            "threat:triage:rules:0:score": 10,
            "timestamp": 1504708744000
          },
          "score": 1,
          "index": "snort_index_2017.09.06.14"
        },
        {
          "id": "d414a9f2-7a03-4e47-9108-16b6fc800dd3",
          "source": {
            "average": 10,
            "min": 10,
            "median": 10,
            "alert": [
              {
                "msg": "snort test alert",
                "threat:triage:rules:0:reason": null,
                "ip_dst_port": "8080",
                "ethsrc": "0A:00:27:00:00:00",
                "threat:triage:score": 10,
                "is_alert": "true",
                "protocol": "TCP",
                "source:type": "snort",
                "ip_dst_addr": "192.168.66.121",
                "ip_src_port": "50187",
                "guid": "snort_test_1",
                "ip_src_addr": "192.168.66.1",
                "threat:triage:rules:0:score": 10,
                "timestamp": 1504708744000
              },
              {
                "msg": "snort test alert 2",
                "threat:triage:rules:0:reason": null,
                "ip_dst_port": "8080",
                "ethsrc": "0A:00:27:00:00:00",
                "threat:triage:score": 10,
                "is_alert": "true",
                "protocol": "TCP",
                "source:type": "snort",
                "ip_dst_addr": "192.168.66.121",
                "ip_src_port": "50187",
                "guid": "snort_test_2",
                "ip_src_addr": "192.168.66.1",
                "threat:triage:rules:0:score": 10,
                "timestamp": 1504708744000
              }
            ],
            "max": 10,
            "threat:triage:score": 20,
            "count": 2,
            "guid": "d414a9f2-7a03-4e47-9108-16b6fc800dd3",
            "groups": [
              "group_one",
              "group_two"
            ],
            "sum": 20,
            "status": "active"
          },
          "score": 1,
          "index": "metaalerts"
        }
      ]
    }
    ```



---

Mime
View raw message