roller-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From agillil...@apache.org
Subject svn commit: r517463 - in /incubator/roller/trunk/apps/planet: src/java/org/apache/roller/planet/business/ src/java/org/apache/roller/planet/business/hibernate/ src/java/org/apache/roller/planet/tasks/ src/java/org/apache/roller/planet/ui/core/ test/jav...
Date Mon, 12 Mar 2007 23:47:49 GMT
Author: agilliland
Date: Mon Mar 12 16:47:48 2007
New Revision: 517463

URL: http://svn.apache.org/viewvc?view=rev&rev=517463
Log:
third pass at Planet cleanup work.  moved the logic around feed fetching into it's own pluggable
interface/implementation classes called FeedFetcher and removed if from PlanetManager.  updated
unit tests accordingly.


Added:
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
    incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java
      - copied, changed from r517436, incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetManagerTest.java
Removed:
    incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetManagerTest.java
Modified:
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/PlanetManager.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetManagerImpl.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
    incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/ui/core/PlanetContext.java
    incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
    incubator/roller/trunk/apps/planet/web/WEB-INF/classes/planet.properties

Added: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java?view=auto&rev=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
(added)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/FeedFetcher.java
Mon Mar 12 16:47:48 2007
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business;
+
+import org.apache.roller.RollerException;
+
+
+/**
+ * A FeedFetcher is what is responsible for actually fetching subscriptions from
+ * their urls and persisting them to the db for the application to make use of.
+ */
+public interface FeedFetcher {
+    
+    /** 
+     * Refresh all subscriptions by fetching and parsing feeds. 
+     */
+    public void refreshEntries(String cacheDirPath) throws RollerException;
+    
+}

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/Planet.java
Mon Mar 12 16:47:48 2007
@@ -52,6 +52,18 @@
     
     
     /**
+     * Get the configured FeedFetcher.
+     */
+    public FeedFetcher getFeedFetcher();
+    
+    
+    /**
+     * Set the configured FeedFetcher.
+     */
+    public void setFeedFetcher(FeedFetcher feedFetcher);
+    
+    
+    /**
      * Flush object states.
      */
     public void flush() throws RollerException;

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/PlanetManager.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/PlanetManager.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/PlanetManager.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/PlanetManager.java
Mon Mar 12 16:47:48 2007
@@ -181,10 +181,4 @@
                            int offset, 
                            int len) throws RollerException;
     
-    
-    /** 
-     * Refresh entry data by fetching and parsing feeds. 
-     */
-    public void refreshEntries(String cacheDirPath) throws RollerException;
-    
 }

Added: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java?view=auto&rev=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
(added)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/RomeFeedFetcher.java
Mon Mar 12 16:47:48 2007
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+package org.apache.roller.planet.business;
+
+import com.sun.syndication.feed.synd.SyndEntry;
+import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.fetcher.impl.FeedFetcherCache;
+import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
+import com.sun.syndication.fetcher.impl.SyndFeedInfo;
+import java.io.File;
+import java.net.URL;
+import java.sql.Timestamp;
+import java.text.MessageFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.roller.RollerException;
+import org.apache.roller.planet.business.hibernate.HibernatePlanetManagerImpl;
+import org.apache.roller.planet.config.PlanetRuntimeConfig;
+import org.apache.roller.planet.pojos.PlanetEntryData;
+import org.apache.roller.planet.pojos.PlanetSubscriptionData;
+import org.apache.roller.planet.util.rome.DiskFeedInfoCache;
+
+
+/**
+ * A FeedFetcher based on the 'Rome' subscription parsing library.
+ */
+public class RomeFeedFetcher implements FeedFetcher {
+    
+    private static Log log = LogFactory.getLog(HibernatePlanetManagerImpl.class);
+    
+    
+    public RomeFeedFetcher() {}
+    
+    
+    // refresh Entries for all Subscriptions
+    public void refreshEntries(String cacheDirPath) throws RollerException {
+        
+        PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+        
+        Date now = new Date();
+        long startTime = System.currentTimeMillis();
+        
+        // can't continue without cache dir
+        if (cacheDirPath == null) {
+            log.warn("Planet cache directory not set, aborting refresh");
+            return;
+        }
+        
+        // allow ${user.home} in cache dir property
+        String cacheDirName = cacheDirPath.replaceFirst(
+                "\\$\\{user.home}",System.getProperty("user.home"));
+        
+        // allow ${catalina.home} in cache dir property
+        if (System.getProperty("catalina.home") != null) {
+            cacheDirName = cacheDirName.replaceFirst(
+                "\\$\\{catalina.home}",System.getProperty("catalina.home"));
+        }
+        
+        // create cache  dir if it does not exist
+        File cacheDir = null;
+        try {
+            cacheDir = new File(cacheDirName);
+            if (!cacheDir.exists()) cacheDir.mkdirs();
+        } catch (Exception e) {
+            log.error("Unable to create planet cache directory: " + cacheDir.getPath(), e);
+            return;
+        }
+        
+        // abort if cache dir is not writable
+        if (!cacheDir.canWrite()) {
+            log.error("Planet cache directory is not writable: " + cacheDir.getPath());
+            return;
+        }
+        
+        FeedFetcherCache feedInfoCache =
+                new DiskFeedInfoCache(cacheDirName);
+        
+        String proxyHost = PlanetRuntimeConfig.getProperty("site.proxyhost");
+        int proxyPort = PlanetRuntimeConfig.getIntProperty("site.proxyport");
+        if (proxyHost != null && proxyPort > 0) {
+            System.setProperty("proxySet", "true");
+            System.setProperty("http.proxyHost", proxyHost);
+            System.setProperty("http.proxyPort", Integer.toString(proxyPort));
+        }
+        /** a hack to set 15 sec timeouts for java.net.HttpURLConnection */
+        System.setProperty("sun.net.client.defaultConnectTimeout", "15000");
+        System.setProperty("sun.net.client.defaultReadTimeout", "15000");
+        
+        com.sun.syndication.fetcher.FeedFetcher feedFetcher = 
+                new HttpURLFeedFetcher(feedInfoCache);
+        //FeedFetcher feedFetcher = new HttpClientFeedFetcher(feedInfoCache);
+        feedFetcher.setUsingDeltaEncoding(false);
+        feedFetcher.setUserAgent("RollerPlanetAggregator");
+        
+        // Loop through all subscriptions in the system
+        Iterator subs = mgr.getSubscriptions().iterator();
+        while (subs.hasNext()) {
+            
+            long subStartTime = System.currentTimeMillis();
+            
+            PlanetSubscriptionData sub = (PlanetSubscriptionData)subs.next();
+            
+            // reattach sub.  sub gets detached as we iterate
+            sub = mgr.getSubscriptionById(sub.getId());
+            
+            Set newEntries = this.getNewEntries(sub, feedFetcher, feedInfoCache);
+            int count = newEntries.size();
+            
+            log.debug("   Entry count: " + count);
+            if (count > 0) {
+                mgr.deleteEntries(sub);
+                sub.addEntries(newEntries);
+                mgr.saveSubscription(sub);
+            }
+            
+            long subEndTime = System.currentTimeMillis();
+            log.debug("   " + count + " - "
+                    + ((subEndTime-subStartTime)/1000.0)
+                    + " seconds to process (" + count + ") entries of "
+                    + sub.getFeedURL());
+        }
+        
+        long endTime = System.currentTimeMillis();
+        log.info("--- DONE --- Refreshed entries in "
+                + ((endTime-startTime)/1000.0) + " seconds");
+    }
+    
+    
+    // get new Entries for a specific Subscription
+    protected Set getNewEntries(PlanetSubscriptionData sub,
+                                com.sun.syndication.fetcher.FeedFetcher feedFetcher,
+                                FeedFetcherCache feedInfoCache)
+            throws RollerException {
+        
+        Set newEntries = new TreeSet();
+        SyndFeed feed = null;
+        URL feedURL = null;
+        Date lastUpdated = new Date();
+        try {
+            feedURL = new URL(sub.getFeedURL());
+            log.debug("Get feed from cache "+sub.getFeedURL());
+            feed = feedFetcher.retrieveFeed(feedURL);
+            SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedURL);
+            if (feedInfo.getLastModified() != null) {
+                long lastUpdatedLong =
+                        ((Long)feedInfo.getLastModified()).longValue();
+                if (lastUpdatedLong != 0) {
+                    lastUpdated = new Date(lastUpdatedLong);
+                }
+            }
+            Thread.sleep(100); // be nice
+        } catch (Exception e) {
+            log.warn("ERROR parsing " + sub.getFeedURL()
+            + " : " + e.getClass().getName() + " : " + e.getMessage());
+            log.debug(e);
+            return newEntries; // bail out
+        }
+        if (lastUpdated!=null && sub.getLastUpdated()!=null) {
+            Calendar feedCal = Calendar.getInstance();
+            feedCal.setTime(lastUpdated);
+            
+            Calendar subCal = Calendar.getInstance();
+            subCal.setTime(sub.getLastUpdated());
+            
+            if (!feedCal.after(subCal)) {
+                if (log.isDebugEnabled()) {
+                    String msg = MessageFormat.format(
+                            "   Skipping ({0} / {1})",
+                            new Object[] {
+                        lastUpdated, sub.getLastUpdated()});
+                    log.debug(msg);
+                }
+                return newEntries; // bail out
+            }
+        }
+        if (feed.getPublishedDate() != null) {
+            sub.setLastUpdated(feed.getPublishedDate());
+            // saving sub here causes detachment issues, so we save it later
+        }
+        
+        // Horrible kludge for Feeds without entry dates: most recent entry is 
+        // given feed's last publish date (or yesterday if none exists) and 
+        // earler entries are placed at once day intervals before that.
+        Calendar cal = Calendar.getInstance();
+        if (sub.getLastUpdated() != null) {
+            cal.setTime(sub.getLastUpdated());
+        } else {
+            cal.setTime(new Date());
+            cal.add(Calendar.DATE, -1);
+        }
+        
+        // Populate subscription object with new entries
+        Iterator entries = feed.getEntries().iterator();
+        while (entries.hasNext()) {
+            try {
+                SyndEntry romeEntry = (SyndEntry) entries.next();
+                PlanetEntryData entry =
+                        new PlanetEntryData(feed, romeEntry, sub);
+                log.debug("Entry title=" + entry.getTitle() + " content size=" + entry.getContent().length());
+                if (entry.getPubTime() == null) {
+                    log.debug("No published date, assigning fake date for "+feedURL);
+                    entry.setPubTime(new Timestamp(cal.getTimeInMillis()));
+                }
+                if (entry.getPermalink() == null) {
+                    log.warn("No permalink, rejecting entry from "+feedURL);
+                } else {
+                    newEntries.add(entry);
+                }
+                cal.add(Calendar.DATE, -1);
+            } catch (Exception e) {
+                log.error("ERROR processing subscription entry", e);
+            }
+        }
+        return newEntries;
+    }
+    
+}

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetImpl.java
Mon Mar 12 16:47:48 2007
@@ -1,20 +1,20 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-*  contributor license agreements.  The ASF licenses this file to You
-* under the Apache License, Version 2.0 (the "License"); you may not
-* use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.  For additional information regarding
-* copyright in this work, please see the NOTICE file in the top level
-* directory of this distribution.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
 
 package org.apache.roller.planet.business.hibernate;
 
@@ -22,6 +22,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.roller.RollerException;
+import org.apache.roller.planet.business.FeedFetcher;
 import org.apache.roller.planet.config.PlanetConfig;
 import org.apache.roller.planet.business.Planet;
 import org.apache.roller.planet.business.PlanetManager;
@@ -49,6 +50,9 @@
     // url strategy
     protected URLStrategy urlStrategy = null;
     
+    // feed fetcher
+    protected FeedFetcher feedFetcher = null;
+    
         
     /**
      * Create HibernatePlanetImpl using Hibernate XML config file or config
@@ -115,6 +119,16 @@
     public void setURLStrategy(URLStrategy urlStrategy) {
         this.urlStrategy = urlStrategy;
         log.info("Using URLStrategy: " + urlStrategy.getClass().getName());
+    }
+    
+    
+    public FeedFetcher getFeedFetcher() {
+        return this.feedFetcher;
+    }
+    
+    public void setFeedFetcher(FeedFetcher feedFetcher) {
+        this.feedFetcher = feedFetcher;
+        log.info("Using FeedFetcher: " + feedFetcher.getClass().getName());
     }
     
     

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetManagerImpl.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetManagerImpl.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetManagerImpl.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/business/hibernate/HibernatePlanetManagerImpl.java
Mon Mar 12 16:47:48 2007
@@ -16,22 +16,9 @@
 
 package org.apache.roller.planet.business.hibernate;
 
-import com.sun.syndication.feed.synd.SyndEntry;
-import com.sun.syndication.feed.synd.SyndFeed;
-import com.sun.syndication.fetcher.FeedFetcher;
-import com.sun.syndication.fetcher.impl.FeedFetcherCache;
-import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
-import com.sun.syndication.fetcher.impl.SyndFeedInfo;
-import java.io.File;
-import java.net.URL;
-import java.sql.Timestamp;
-import java.text.MessageFormat;
-import java.util.Calendar;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
 import org.hibernate.Criteria;
 import org.hibernate.HibernateException;
 import org.hibernate.Query;
@@ -43,12 +30,10 @@
 import org.apache.roller.RollerException;
 import org.apache.roller.planet.business.AbstractManagerImpl;
 import org.apache.roller.planet.business.PlanetManager;
-import org.apache.roller.planet.config.PlanetRuntimeConfig;
 import org.apache.roller.planet.pojos.PlanetData;
 import org.apache.roller.planet.pojos.PlanetEntryData;
 import org.apache.roller.planet.pojos.PlanetGroupData;
 import org.apache.roller.planet.pojos.PlanetSubscriptionData;
-import org.apache.roller.planet.util.rome.DiskFeedInfoCache;
 
 
 /**
@@ -86,7 +71,6 @@
             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
             Criteria criteria = session.createCriteria(PlanetData.class);
             criteria.add(Expression.ilike("handle", handle));
-            criteria.setMaxResults(1);
             planet = (PlanetData) criteria.uniqueResult();
         } catch (HibernateException e) {
             throw new RollerException(e);
@@ -145,7 +129,6 @@
             Criteria criteria = session.createCriteria(PlanetGroupData.class);
             criteria.add(Expression.eq("planet", planet));
             criteria.add(Expression.eq("handle", handle));
-            criteria.setMaxResults(1);
             return (PlanetGroupData) criteria.uniqueResult();
         } catch (HibernateException e) {
             throw new RollerException(e);
@@ -186,7 +169,6 @@
             Criteria criteria =
                     session.createCriteria(PlanetSubscriptionData.class);
             criteria.add(Expression.eq("feedURL", feedURL));
-            criteria.setMaxResults(1);
             return (PlanetSubscriptionData) criteria.uniqueResult();
         } catch (HibernateException e) {
             throw new RollerException(e);
@@ -408,186 +390,6 @@
         }
         
         return ret;
-    }
-    
-    
-    // refresh Entries for all Subscriptions
-    public void refreshEntries(String cacheDirPath) throws RollerException {
-        
-        Date now = new Date();
-        long startTime = System.currentTimeMillis();
-        
-        // can't continue without cache dir
-        if (cacheDirPath == null) {
-            log.warn("Planet cache directory not set, aborting refresh");
-            return;
-        }
-        
-        // allow ${user.home} in cache dir property
-        String cacheDirName = cacheDirPath.replaceFirst(
-                "\\$\\{user.home}",System.getProperty("user.home"));
-        
-        // allow ${catalina.home} in cache dir property
-        if (System.getProperty("catalina.home") != null) {
-            cacheDirName = cacheDirName.replaceFirst(
-                "\\$\\{catalina.home}",System.getProperty("catalina.home"));
-        }
-        
-        // create cache  dir if it does not exist
-        File cacheDir = null;
-        try {
-            cacheDir = new File(cacheDirName);
-            if (!cacheDir.exists()) cacheDir.mkdirs();
-        } catch (Exception e) {
-            log.error("Unable to create planet cache directory: " + cacheDir.getPath(), e);
-            return;
-        }
-        
-        // abort if cache dir is not writable
-        if (!cacheDir.canWrite()) {
-            log.error("Planet cache directory is not writable: " + cacheDir.getPath());
-            return;
-        }
-        
-        FeedFetcherCache feedInfoCache =
-                new DiskFeedInfoCache(cacheDirName);
-        
-        String proxyHost = PlanetRuntimeConfig.getProperty("site.proxyhost");
-        int proxyPort = PlanetRuntimeConfig.getIntProperty("site.proxyport");
-        if (proxyHost != null && proxyPort > 0) {
-            System.setProperty("proxySet", "true");
-            System.setProperty("http.proxyHost", proxyHost);
-            System.setProperty("http.proxyPort", Integer.toString(proxyPort));
-        }
-        /** a hack to set 15 sec timeouts for java.net.HttpURLConnection */
-        System.setProperty("sun.net.client.defaultConnectTimeout", "15000");
-        System.setProperty("sun.net.client.defaultReadTimeout", "15000");
-        
-        FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache);
-        //FeedFetcher feedFetcher = new HttpClientFeedFetcher(feedInfoCache);
-        feedFetcher.setUsingDeltaEncoding(false);
-        feedFetcher.setUserAgent("RollerPlanetAggregator");
-        
-        // Loop through all subscriptions in the system
-        Iterator subs = getSubscriptions().iterator();
-        while (subs.hasNext()) {
-            
-            long subStartTime = System.currentTimeMillis();
-            
-            PlanetSubscriptionData sub = (PlanetSubscriptionData)subs.next();
-            
-            // reattach sub.  sub gets detached as we iterate
-            sub = this.getSubscriptionById(sub.getId());
-            
-            Set newEntries = this.getNewEntries(sub, feedFetcher, feedInfoCache);
-            int count = newEntries.size();
-            
-            log.debug("   Entry count: " + count);
-            if (count > 0) {
-                this.deleteEntries(sub);
-                sub.addEntries(newEntries);
-                this.saveSubscription(sub);
-                this.strategy.flush();
-            }
-            long subEndTime = System.currentTimeMillis();
-            log.debug("   " + count + " - "
-                    + ((subEndTime-subStartTime)/1000.0)
-                    + " seconds to process (" + count + ") entries of "
-                    + sub.getFeedURL());
-        }
-        
-        long endTime = System.currentTimeMillis();
-        log.info("--- DONE --- Refreshed entries in "
-                + ((endTime-startTime)/1000.0) + " seconds");
-    }
-    
-    
-    // get new Entries for a specific Subscription
-    protected Set getNewEntries(PlanetSubscriptionData sub,
-                                FeedFetcher feedFetcher,
-                                FeedFetcherCache feedInfoCache)
-            throws RollerException {
-        
-        Set newEntries = new TreeSet();
-        SyndFeed feed = null;
-        URL feedURL = null;
-        Date lastUpdated = new Date();
-        try {
-            feedURL = new URL(sub.getFeedURL());
-            log.debug("Get feed from cache "+sub.getFeedURL());
-            feed = feedFetcher.retrieveFeed(feedURL);
-            SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedURL);
-            if (feedInfo.getLastModified() != null) {
-                long lastUpdatedLong =
-                        ((Long)feedInfo.getLastModified()).longValue();
-                if (lastUpdatedLong != 0) {
-                    lastUpdated = new Date(lastUpdatedLong);
-                }
-            }
-            Thread.sleep(100); // be nice
-        } catch (Exception e) {
-            log.warn("ERROR parsing " + sub.getFeedURL()
-            + " : " + e.getClass().getName() + " : " + e.getMessage());
-            log.debug(e);
-            return newEntries; // bail out
-        }
-        if (lastUpdated!=null && sub.getLastUpdated()!=null) {
-            Calendar feedCal = Calendar.getInstance();
-            feedCal.setTime(lastUpdated);
-            
-            Calendar subCal = Calendar.getInstance();
-            subCal.setTime(sub.getLastUpdated());
-            
-            if (!feedCal.after(subCal)) {
-                if (log.isDebugEnabled()) {
-                    String msg = MessageFormat.format(
-                            "   Skipping ({0} / {1})",
-                            new Object[] {
-                        lastUpdated, sub.getLastUpdated()});
-                    log.debug(msg);
-                }
-                return newEntries; // bail out
-            }
-        }
-        if (feed.getPublishedDate() != null) {
-            sub.setLastUpdated(feed.getPublishedDate());
-            // saving sub here causes detachment issues, so we save it later
-        }
-        
-        // Horrible kludge for Feeds without entry dates: most recent entry is 
-        // given feed's last publish date (or yesterday if none exists) and 
-        // earler entries are placed at once day intervals before that.
-        Calendar cal = Calendar.getInstance();
-        if (sub.getLastUpdated() != null) {
-            cal.setTime(sub.getLastUpdated());
-        } else {
-            cal.setTime(new Date());
-            cal.add(Calendar.DATE, -1);
-        }
-        
-        // Populate subscription object with new entries
-        Iterator entries = feed.getEntries().iterator();
-        while (entries.hasNext()) {
-            try {
-                SyndEntry romeEntry = (SyndEntry) entries.next();
-                PlanetEntryData entry =
-                        new PlanetEntryData(feed, romeEntry, sub);
-                log.debug("Entry title=" + entry.getTitle() + " content size=" + entry.getContent().length());
-                if (entry.getPubTime() == null) {
-                    log.debug("No published date, assigning fake date for "+feedURL);
-                    entry.setPubTime(new Timestamp(cal.getTimeInMillis()));
-                }
-                if (entry.getPermalink() == null) {
-                    log.warn("No permalink, rejecting entry from "+feedURL);
-                } else {
-                    newEntries.add(entry);
-                }
-                cal.add(Calendar.DATE, -1);
-            } catch (Exception e) {
-                log.error("ERROR processing subscription entry", e);
-            }
-        }
-        return newEntries;
     }
 
 }

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/tasks/RefreshPlanetTask.java
Mon Mar 12 16:47:48 2007
@@ -41,7 +41,7 @@
             // Update all feeds in planet
             log.info("Refreshing Planet entries");
             Planet planet = PlanetFactory.getPlanet();
-            planet.getPlanetManager().refreshEntries(
+            planet.getFeedFetcher().refreshEntries(
                 PlanetConfig.getProperty("cache.dir"));                        
             planet.flush();
             planet.release();

Modified: incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/ui/core/PlanetContext.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/ui/core/PlanetContext.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/ui/core/PlanetContext.java
(original)
+++ incubator/roller/trunk/apps/planet/src/java/org/apache/roller/planet/ui/core/PlanetContext.java
Mon Mar 12 16:47:48 2007
@@ -24,6 +24,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.roller.RollerException;
+import org.apache.roller.planet.business.FeedFetcher;
 import org.apache.roller.planet.business.Planet;
 import org.apache.roller.planet.business.PlanetFactory;
 import org.apache.roller.planet.business.URLStrategy;
@@ -64,7 +65,7 @@
      */
     public void contextInitialized(ServletContextEvent sce) {
         
-        log.debug("Roller Planet Initializing");
+        log.debug("Roller Planet Initializing ... ");
         
         // keep a reverence to ServletContext object
         this.context = sce.getServletContext();
@@ -83,6 +84,7 @@
             
             setupRuntimeProperties();
             setupURLStrategy();
+            setupFeedFetcher();
             
             planet.flush();
             planet.release();
@@ -92,7 +94,7 @@
             throw new RuntimeException(t);
         }
         
-        log.debug("Initialization Complete");
+        log.debug("Roller Planet Initialization Complete");
     }
     
     
@@ -127,6 +129,26 @@
         
         // plug it in
         PlanetFactory.getPlanet().setURLStrategy(urlStrategy);
+    }
+    
+    
+    /**
+     * Lookup configured FeedFetcher from config and plug it in.
+     *
+     * If no FeedFetcher can be configured then we bail and spew errors.
+     */
+    private void setupFeedFetcher() throws Exception {
+        
+        String feedFetchClass = PlanetConfig.getProperty("feedfetcher.classname");
+        if(feedFetchClass == null || feedFetchClass.trim().length() < 1) {
+            throw new Exception("No FeedFetcher configured!!!");
+        }
+        
+        Class fetchClass = Class.forName(feedFetchClass);
+        FeedFetcher feedFetcher = (FeedFetcher) fetchClass.newInstance();
+        
+        // plug it in
+        PlanetFactory.getPlanet().setFeedFetcher(feedFetcher);
     }
     
     

Modified: incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
(original)
+++ incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetTestSuite.java
Mon Mar 12 16:47:48 2007
@@ -59,7 +59,7 @@
         suite.addTestSuite(EntryBasicTests.class);
         suite.addTestSuite(EntryFunctionalTests.class);
         
-        suite.addTestSuite(PlanetManagerTest.class);
+        suite.addTestSuite(RomeFeedFetcherTest.class);
 
         return suite;
     }

Copied: incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java
(from r517436, incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetManagerTest.java)
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java?view=diff&rev=517463&p1=incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetManagerTest.java&r1=517436&p2=incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/PlanetManagerTest.java
(original)
+++ incubator/roller/trunk/apps/planet/test/java/org/apache/roller/planet/business/RomeFeedFetcherTest.java
Mon Mar 12 16:47:48 2007
@@ -17,22 +17,11 @@
 package org.apache.roller.planet.business;
 
 import java.io.File;
-import java.sql.Timestamp;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import junit.framework.Test;
 import junit.framework.TestCase;
-import junit.framework.TestSuite;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.roller.planet.TestUtils;
-import org.apache.roller.planet.business.Planet;
-import org.apache.roller.planet.business.PlanetFactory;
-import org.apache.roller.planet.business.PlanetManager;
 import org.apache.roller.planet.pojos.PlanetData;
-import org.apache.roller.planet.pojos.PlanetEntryData;
 import org.apache.roller.planet.pojos.PlanetGroupData;
 import org.apache.roller.planet.pojos.PlanetSubscriptionData;
 
@@ -40,136 +29,53 @@
 /**
  * Test database implementation of PlanetManager.
  */
-public class PlanetManagerTest extends TestCase {
+public class RomeFeedFetcherTest extends TestCase {
+    
+    public static Log log = LogFactory.getLog(RomeFeedFetcherTest.class);   
     
-    public static Log log = LogFactory.getLog(PlanetManagerTest.class);   
     private PlanetData testPlanet = null;
+    private PlanetGroupData testGroup = null;
+    private PlanetSubscriptionData testSub = null;
+    
+    String feed_url = "http://rollerweblogger.org/roller/feed/entries/rss";
+    
     
-        
     protected void setUp() throws Exception {
-        testPlanet = TestUtils.setupPlanet("groupTestPlanet");
-    }
+        testPlanet = TestUtils.setupPlanet("fetcherTestPlanet");
+        testGroup = TestUtils.setupGroup(testPlanet, "fetcherTestGroup");
+        
+        // add test subscription
+        PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+        testSub = new PlanetSubscriptionData();
+        testSub.setTitle(feed_url);
+        testSub.setFeedURL(feed_url);
+        mgr.saveSubscription(testSub);
+        PlanetFactory.getPlanet().flush();
         
+        // use RomeFeedFetcher
+        PlanetFactory.getPlanet().setFeedFetcher(new RomeFeedFetcher());
+    }
+    
+    
     protected void tearDown() throws Exception {
+        TestUtils.teardownSubscription(testSub.getId());
+        TestUtils.teardownGroup(testGroup.getId());
         TestUtils.teardownPlanet(testPlanet.getId());
     }
     
     
     public void testRefreshEntries() throws Exception {
         
-        PlanetManager planet = PlanetFactory.getPlanet().getPlanetManager();
-        
-        String feed_url1 = "http://rollerweblogger.org/roller/feed/entries/rss";
+        PlanetManager mgr = PlanetFactory.getPlanet().getPlanetManager();
+        FeedFetcher feedFetcher = PlanetFactory.getPlanet().getFeedFetcher();
+
+        // refresh entries
+        feedFetcher.refreshEntries("." + File.separator + "planet-cache");
+        TestUtils.endSession(true);
         
-        {
-//            PlanetConfigData config = planet.getConfiguration();
-//            config.setAdminName("admin");
-//            config.setSiteURL("http://localhost:8080/roller");
-//            planet.saveConfiguration(config);
-            
-            PlanetGroupData group = new PlanetGroupData();
-            group.setDescription("test_group_desc");
-            group.setHandle("test_handle");
-            group.setTitle("test_title");
-            group.setPlanet(testPlanet);
-            planet.saveGroup(group);
-            
-            PlanetSubscriptionData sub = new PlanetSubscriptionData();
-            sub.setTitle(feed_url1);
-            sub.setFeedURL(feed_url1);
-            planet.saveSubscription(sub);
-            
-            group.getSubscriptions().add(sub);            
-            planet.saveGroup(group);
-            TestUtils.endSession(true);
-        }
-        {
-            planet.refreshEntries("." + File.separator + "planet-cache");
-            TestUtils.endSession(true);
-            
-            PlanetSubscriptionData sub = planet.getSubscription(feed_url1);
-            int entriesSize = sub.getEntries().size();
-            
-            PlanetGroupData group = planet.getGroup(testPlanet, "test_handle");
-            assertNotNull(group);
-            
-            planet.deleteGroup(group);
-            planet.deleteSubscription(sub);
-            TestUtils.endSession(true);
-            
-            assertTrue(entriesSize > 0);
-            
-            Object feed = planet.getSubscription(feed_url1);
-            assertNull(feed);
-        }
+        PlanetSubscriptionData sub = mgr.getSubscription(feed_url);
+        assertNotNull(sub);
+        assertTrue(sub.getEntries().size() > 0);
     }
-    
-    
-//    public void _testAggregations() throws Exception {
-//        
-//        try {
-//            PlanetManager planet = PlanetFactory.getPlanet().getPlanetManager();
-//            
-//            String feed_url1 = "http://rollerweblogger.org/roller/feed/entries/rss";
-//            String feed_url2 = "http://blogs.sun.com/main/feed/entries/atom";
-//            
-//            {
-//                PlanetGroupData group = new PlanetGroupData();
-//                group.setDescription("test_group_desc");
-//                group.setHandle("test_handle");
-//                group.setTitle("test_title");
-//                group.setPlanet(testPlanet);
-//                planet.saveGroup(group);
-//                
-//                PlanetSubscriptionData sub1 = new PlanetSubscriptionData();
-//                sub1.setTitle(feed_url1);
-//                sub1.setFeedURL(feed_url1);
-//                planet.saveSubscription(sub1);
-//                
-//                PlanetSubscriptionData sub2 = new PlanetSubscriptionData();
-//                sub2.setTitle(feed_url2);
-//                sub2.setFeedURL(feed_url2);
-//                planet.saveSubscription(sub2);
-//                
-//                group.getSubscriptions().add(sub1);
-//                group.getSubscriptions().add(sub2);
-//                planet.saveGroup(group);
-//                TestUtils.endSession(true);
-//            }
-//            {
-//                planet.refreshEntries(null);
-//                TestUtils.endSession(true);
-//                
-//                int count = 0;
-//                Iterator subs = planet.getSubscriptions().iterator();
-//                while  (subs.hasNext()) {
-//                    PlanetSubscriptionData sub= (PlanetSubscriptionData)subs.next();
-//                    count += sub.getEntries().size();
-//                }
-//                PlanetSubscriptionData sub1 = planet.getSubscription(feed_url1);
-//                assertTrue(sub1.getEntries().size() > 0);
-//                PlanetSubscriptionData sub2 = planet.getSubscription(feed_url2);
-//                assertTrue(sub2.getEntries().size() > 0);
-//                assertEquals(count, sub1.getEntries().size() + sub2.getEntries().size());
-//                
-//                PlanetGroupData group = planet.getGroup(testPlanet, "test_handle");
-//                assertNotNull(group);
-//                
-//                List bigag = planet.getEntries(group, null, null, 0, 30);
-//                assertEquals(30, bigag.size());
-//                
-//                List littleag = planet.getEntries(group, null, null, 0, 10);
-//                assertEquals(10, littleag.size());
-//                
-//                planet.deleteGroup(group);
-//                planet.deleteSubscription(sub1);
-//                planet.deleteSubscription(sub2);
-//                TestUtils.endSession(true);
-//            }
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//            fail();
-//        }
-//    }
     
 }

Modified: incubator/roller/trunk/apps/planet/web/WEB-INF/classes/planet.properties
URL: http://svn.apache.org/viewvc/incubator/roller/trunk/apps/planet/web/WEB-INF/classes/planet.properties?view=diff&rev=517463&r1=517462&r2=517463
==============================================================================
--- incubator/roller/trunk/apps/planet/web/WEB-INF/classes/planet.properties (original)
+++ incubator/roller/trunk/apps/planet/web/WEB-INF/classes/planet.properties Mon Mar 12 16:47:48
2007
@@ -56,6 +56,10 @@
 urlstrategy.classname=\
 org.apache.roller.planet.business.MultiPlanetURLStrategy
 
+# choose a feed fetcher
+feedfetcher.classname=\
+org.apache.roller.planet.business.RomeFeedFetcher
+
 # The set of default Roller renderer factories.
 rendering.rollerRendererFactories=\
 org.apache.roller.planet.ui.rendering.velocity.VelocityRendererFactory



Mime
View raw message