metron-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickwallen <...@git.apache.org>
Subject [GitHub] metron pull request #613: METRON-990: Clean up and organize flux properties
Date Wed, 21 Jun 2017 21:40:34 GMT
Github user nickwallen commented on a diff in the pull request:

    https://github.com/apache/metron/pull/613#discussion_r123373290
  
    --- Diff: metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
---
    @@ -253,69 +253,295 @@
             </value-attributes>
         </property>
         <property>
    -        <name>global-json</name>
    -        <display-name>global.json template</display-name>
    -        <description>This is the jinja template for global.json file</description>
    -        <value>
    -{
    -"es.clustername": "{{ es_cluster_name }}",
    -"es.ip": "{{ es_url }}",
    -"es.date.format": "yyyy.MM.dd.HH",
    -"parser.error.topic": "indexing"
    -}
    -        </value>
    +        <name>es_date_format</name>
    +        <description>Elasticsearch Date Format</description>
    +        <value>yyyy.MM.dd.HH</value>
    +        <display-name>Elasticsearch Date Format</display-name>
    +    </property>
    +    <property>
    +        <name>parser_error_topic</name>
    +        <description>Parser Error Topic</description>
    +        <value>indexing</value>
    +        <display-name>Parser Error Topic</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_workers</name>
    +        <description>Number of Enrichment Topology Workers</description>
    +        <value>1</value>
    +        <display-name>Enrichment Workers</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_acker_executors</name>
    +        <description>Number of Enrichment Topology Ackers</description>
    +        <value>1</value>
    +        <display-name>Enrichment Ackers</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_topology_worker_childopts</name>
    +        <description>Enrichment Topology JVM Options</description>
    +        <value/>
    +        <display-name>Enrichment childopts</display-name>
             <value-attributes>
    -            <type>content</type>
    +            <empty-value-valid>true</empty-value-valid>
    +        </value-attributes>
    +    </property>
    +    <property>
    +        <name>enrichment_topology_max_spout_pending</name>
    +        <description>Enrichment Topology Spout Max Pending Tuples</description>
    +        <value/>
    +        <display-name>Enrichment Max Pending</display-name>
    +        <value-attributes>
    +            <empty-value-valid>true</empty-value-valid>
             </value-attributes>
         </property>
         <property>
    -        <name>elasticsearch-properties</name>
    -        <description>The template for the elasticsearch.properties file.</description>
    -        <display-name>elasticsearch.properties template</display-name>
    -        <value>
    -##### Storm #####
    -indexing.workers=1
    -indexing.executors=0
    -topology.worker.childopts={{topology_worker_childopts}}
    -topology.auto-credentials={{topology_auto_credentials}}
    -##### Kafka #####
    -kafka.zk={{ zookeeper_quorum }}
    -kafka.broker={{ kafka_brokers }}
    -kafka.start=UNCOMMITTED_EARLIEST
    -kafka.security.protocol={{kafka_security_protocol}}
    -storm.auto.credentials=[]
    -##### Indexing #####
    -index.input.topic=indexing
    -index.error.topic=indexing
    -writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
    -##### Metrics #####
    -#reporters
    -org.apache.metron.metrics.reporter.graphite=true
    -org.apache.metron.metrics.reporter.console=false
    -org.apache.metron.metrics.reporter.jmx=false
    -#Graphite Addresses
    -org.apache.metron.metrics.graphite.address=localhost
    -org.apache.metron.metrics.graphite.port=2023
    -#TelemetryParserBolt
    -org.apache.metron.metrics.TelemetryParserBolt.acks=true
    -org.apache.metron.metrics.TelemetryParserBolt.emits=true
    -org.apache.metron.metrics.TelemetryParserBolt.fails=true
    -##### HDFS #####
    -bolt.hdfs.batch.size=5000
    -bolt.hdfs.field.delimiter=|
    -bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
    -bolt.hdfs.rotation.policy.units=DAYS
    -bolt.hdfs.rotation.policy.count=1
    -bolt.hdfs.file.rotation.size.in.mb=5
    -bolt.hdfs.file.system.url={{ default_fs }}
    -bolt.hdfs.wip.file.path=/paloalto/wip
    -bolt.hdfs.finished.file.path=/paloalto/rotated
    -bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
    -index.hdfs.output={{ metron_apps_indexed_hdfs_dir }}
    -        </value>
    +        <name>enrichment_kafka_start</name>
    +        <description>Enrichment Topology Spout Offset</description>
    +        <value>UNCOMMITTED_EARLIEST</value>
    +        <display-name>Enrichment Offset</display-name>
    +        <value-attributes>
    +            <type>value-list</type>
    +            <entries>
    +                <entry>
    +                    <value>EARLIEST</value>
    +                </entry>
    +                <entry>
    +                    <value>LATEST</value>
    +                </entry>
    +                <entry>
    +                    <value>UNCOMMITTED_EARLIEST</value>
    +                </entry>
    +                <entry>
    +                    <value>UNCOMMITTED_LATEST</value>
    +                </entry>
    +            </entries>
    +            <selection-cardinality>1</selection-cardinality>
    +        </value-attributes>
    +    </property>
    +    <property>
    +        <name>enrichment_input_topic</name>
    +        <description>Enrichment Input Topic</description>
    +        <value>enrichments</value>
    +        <display-name>Enrichment Input Topic</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_output_topic</name>
    +        <description>Enrichment Output Topic</description>
    +        <value>indexing</value>
    +        <display-name>Enrichment Output Topic</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_error_topic</name>
    +        <description>Enrichment Error Topic</description>
    +        <value>indexing</value>
    +        <display-name>Enrichment Error Topic</display-name>
    +    </property>
    +    <property>
    +        <name>threatintel_error_topic</name>
    +        <description>Threat Intel Error Topic</description>
    +        <value>indexing</value>
    +        <display-name>Threat Intel Error Topic</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_join_cache_size</name>
    +        <description>Enrichment Join Bolt Cache Max Size</description>
    +        <value>100000</value>
    +        <display-name>Enrichment Join Size</display-name>
    +    </property>
    +    <property>
    +        <name>threatintel_join_cache_size</name>
    +        <description>Threat Intel Join Bolt Cache Max Size</description>
    +        <value>100000</value>
    +        <display-name>Threat Intel Join Size</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_host_known_hosts</name>
    +        <description>List of Known Hosts for Host Enrichment</description>
    +        <value>[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value"
: "important"},{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" :
"important"},{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]</value>
    +        <display-name>Host Enrichment</display-name>
             <value-attributes>
                 <type>content</type>
             </value-attributes>
    -        <on-ambari-upgrade add="true"/>
    +    </property>
    +    <property>
    +        <name>enrichment_kafka_spout_parallelism</name>
    +        <description>Enrichment Topology Kafka Spout Parallelism</description>
    +        <value>1</value>
    +        <display-name>Enrichment Spout Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_split_parallelism</name>
    +        <description>Enrichment Topology Enrichment Split Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Enrichment Split Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_stellar_parallelism</name>
    +        <description>Enrichment Topology Enrichment Stellar Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Enrichment Stellar Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>enrichment_join_parallelism</name>
    +        <description>Enrichment Topology Enrichment Join Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Enrichment Join Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>threat_intel_split_parallelism</name>
    +        <description>Enrichment Topology Threat Intel Split Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Threat Intel Spout Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>threat_intel_stellar_parallelism</name>
    +        <description>Enrichment Topology Threat Intel Stellar Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Threat Intel Spout Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>threat_intel_join_parallelism</name>
    +        <description>Enrichment Topology Threat Intel Join Bolt Parallelism</description>
    +        <value>1</value>
    +        <display-name>Threat Intel Join Parallelism</display-name>
    +    </property>
    +    <property>
    +        <name>kafka_writer_parallelism</name>
    +        <description>Enrichment Topology Kafka Writer Parallelism</description>
    +        <value>1</value>
    +        <display-name>Enrichment Kafka Writer Parallelism</display-name>
    +    </property>
    +
    +    <property>
    +        <name>indexing_workers</name>
    +        <description>Number of Indexing Topology Workers</description>
    +        <value>1</value>
    +        <display-name>Indexing Workers</display-name>
    +    </property>
    +    <property>
    +        <name>indexing_acker_executors</name>
    +        <description>Number of Indexing Topology Ackers</description>
    +        <value>1</value>
    +        <display-name>Enrichment Ackers</display-name>
    +    </property>
    +    <property>
    +        <name>indexing_topology_worker_childopts</name>
    +        <description>Indexing Topology JVM Options</description>
    +        <value/>
    +        <display-name>Indexing childopts</display-name>
    +        <value-attributes>
    +            <empty-value-valid>true</empty-value-valid>
    +        </value-attributes>
    +    </property>
    +    <property>
    +        <name>indexing_topology_max_spout_pending</name>
    +        <description>Indexing Topology Spout Max Pending Tuples</description>
    +        <value/>
    +        <display-name>Indexing Max Pending</display-name>
    +        <value-attributes>
    +            <empty-value-valid>true</empty-value-valid>
    +        </value-attributes>
    +    </property>
    +    <property>
    +        <name>indexing_kafka_start</name>
    +        <description>Indexing Topology Spout Offset</description>
    +        <value>UNCOMMITTED_EARLIEST</value>
    +        <display-name>Indexing Offset</display-name>
    +        <value-attributes>
    +            <type>value-list</type>
    +            <entries>
    +                <entry>
    +                    <value>EARLIEST</value>
    +                </entry>
    +                <entry>
    +                    <value>LATEST</value>
    +                </entry>
    +                <entry>
    +                    <value>UNCOMMITTED_EARLIEST</value>
    +                </entry>
    +                <entry>
    +                    <value>UNCOMMITTED_LATEST</value>
    +                </entry>
    +            </entries>
    +            <selection-cardinality>1</selection-cardinality>
    +        </value-attributes>
    +    </property>
    +    <property>
    +        <name>indexing_input_topic</name>
    +        <description>Indexing Input Topic</description>
    +        <value>indexing</value>
    +        <display-name>Indexing Input Topic</display-name>
    +    </property>
    +    <property>
    +        <name>indexing_error_topic</name>
    +        <description>Indexing Error Topic</description>
    +        <value>indexing</value>
    +        <display-name>Indexing Error Topic</display-name>
    +    </property>
    +    <property>
    +        <name>indexing_writer_class_name</name>
    +        <description>Indexing Writer Class Name</description>
    +        <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value>
    +        <display-name>Indexing Writer Class Name</display-name>
    +    </property>
    +    <property>
    +        <name>bolt_hdfs_rotation_policy</name>
    +        <description>Class Name of HDFS Rotation Policy</description>
    +        <value>org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy</value>
    +        <display-name>HDFS Rotation Policy</display-name>
    +    </property>
    +    <property>
    +        <name>bolt_hdfs_rotation_policy_count</name>
    +        <description>HDFS Rotation Policy Count</description>
    +        <value>1</value>
    +        <display-name>HDFS Rotation Policy Count</display-name>
    +    </property>
    +    <property>
    +        <name>bolt_hdfs_rotation_policy_units</name>
    +        <description>Frequency of HDFS File Rotation</description>
    +        <value>DAYS</value>
    +        <display-name>HDFS Rotation Policy Units</display-name>
    --- End diff --
    
    I think the values would be GB, KB, MB, TB  based on what[ I am reading here.](https://storm.apache.org/releases/1.0.1/javadocs/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.Units.html)


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message