eagle-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (EAGLE-526) aggregation framework for mr history jobs
Date Mon, 12 Sep 2016 06:31:20 GMT

    [ https://issues.apache.org/jira/browse/EAGLE-526?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15483234#comment-15483234
] 

ASF GitHub Bot commented on EAGLE-526:
--------------------------------------

Github user haoch commented on a diff in the pull request:

    https://github.com/apache/incubator-eagle/pull/430#discussion_r78320310
  
    --- Diff: eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
---
    @@ -0,0 +1,137 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    +*/
    +
    +package org.apache.eagle.jpm.aggregation.storm;
    +
    +import org.apache.eagle.jpm.aggregation.AggregationConfig;
    +import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
    +import org.apache.eagle.jpm.aggregation.state.AggregationTimeManager;
    +import org.apache.eagle.jpm.util.Utils;
    +import backtype.storm.spout.SpoutOutputCollector;
    +import backtype.storm.task.TopologyContext;
    +import backtype.storm.topology.OutputFieldsDeclarer;
    +import backtype.storm.topology.base.BaseRichSpout;
    +import backtype.storm.tuple.Fields;
    +import backtype.storm.tuple.Values;
    +import com.typesafe.config.Config;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.util.HashSet;
    +import java.util.Map;
    +import java.util.Set;
    +
    +public class AggregationSpout extends BaseRichSpout {
    +    private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class);
    +    private static final Long MAX_WAIT_TIME = 12 * 60 * 60000L;//12 hours
    +    private static final Long MAX_SAFE_TIME = 6 * 60 * 60000L;//6 hours
    +
    +    private Config config;
    +    MetricsAggregateContainer jobProcessTime;
    +
    +    private SpoutOutputCollector collector;
    +    private Set<Long> processStartTime;
    +    private Long lastUpdateTime;
    +
    +    public AggregationSpout(Config config, MetricsAggregateContainer jobProcessTime)
{
    +        this.config = config;
    +        this.jobProcessTime = jobProcessTime;
    +        this.processStartTime = new HashSet<>();
    +    }
    +
    +    @Override
    +    public void open(Map conf, TopologyContext context, final SpoutOutputCollector collector)
{
    +        AggregationConfig.getInstance(config);
    +        AggregationTimeManager.instance().init(AggregationConfig.get().getZkStateConfig());
    +        this.collector = collector;
    +    }
    +
    +    @Override
    +    public void nextTuple() {
    +        Utils.sleep(5);
    +        LOG.info("start to run");
    --- End diff --
    
    LOG.


> aggregation framework for mr history jobs
> -----------------------------------------
>
>                 Key: EAGLE-526
>                 URL: https://issues.apache.org/jira/browse/EAGLE-526
>             Project: Eagle
>          Issue Type: New Feature
>            Reporter: wujinhu
>            Assignee: wujinhu
>




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message