oodt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bfos...@apache.org
Subject svn commit: r1366499 - in /oodt/trunk/pushpull: ./ src/main/java/org/apache/oodt/cas/pushpull/exceptions/ src/main/java/org/apache/oodt/cas/pushpull/protocol/ src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/ src/main/java/org/apache/oodt/cas...
Date Fri, 27 Jul 2012 19:05:30 GMT
Author: bfoster
Date: Fri Jul 27 19:05:29 2012
New Revision: 1366499

URL: http://svn.apache.org/viewvc?rev=1366499&view=rev
Log:
- Fixed: RemoteSiteFile doesn't always set RemoteSite

---------------
OODT-476

Modified:
    oodt/trunk/pushpull/pom.xml
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
    oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java

Modified: oodt/trunk/pushpull/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/pom.xml?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/pom.xml (original)
+++ oodt/trunk/pushpull/pom.xml Fri Jul 27 19:05:29 2012
@@ -147,6 +147,11 @@
       <version>2.0.8</version>
     </dependency>
     <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>10.0.1</version>
+    </dependency>
+    <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>3.8.2</version>

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
Fri Jul 27 19:05:29 2012
@@ -19,10 +19,10 @@
 package org.apache.oodt.cas.pushpull.exceptions;
 
 /**
- * 
+ *
  * @author bfoster
  * @version $Revision$
- * 
+ *
  * <p>
  * Describe your class here
  * </p>.
@@ -39,6 +39,10 @@ public class PushPullFrameworkException 
         super(PushPullFrameworkException.addCallingClassToMsg(msg));
     }
 
+    public PushPullFrameworkException(String msg, Throwable t) {
+       super(PushPullFrameworkException.addCallingClassToMsg(msg), t);
+    }
+
     private static String addCallingClassToMsg(String msg) {
         try {
             Throwable stack = new Throwable();

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
Fri Jul 27 19:05:29 2012
@@ -19,10 +19,10 @@
 package org.apache.oodt.cas.pushpull.exceptions;
 
 /**
- * 
+ *
  * @author bfoster
  * @version $Revision$
- * 
+ *
  * <p>
  * Describe your class here
  * </p>.
@@ -38,4 +38,8 @@ public class RemoteConnectionException e
     public RemoteConnectionException(String msg) {
         super(msg);
     }
+
+    public RemoteConnectionException(String msg, Throwable t) {
+       super(msg, t);
+   }
 }

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
Fri Jul 27 19:05:29 2012
@@ -52,27 +52,27 @@ import java.util.logging.Logger;
  * 'allowReuse' to false then a new Protocol object will be created and
  * returned.<br>
  * <br>
- * 
+ *
  * @author bfoster
  */
 public class ProtocolHandler {
 
-  private HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
+  private final HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
 
-  private HashMap<URL, Protocol> reuseProtocols;
+  private final HashMap<URL, Protocol> reuseProtocols;
 
-  private HashMap<RemoteSiteFile, PagingInfo> pageInfos;
+  private final HashMap<RemoteSiteFile, PagingInfo> pageInfos;
 
-  private HashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
+  private final HashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
 
-  private ProtocolInfo pi;
+  private final ProtocolInfo pi;
 
   private static final Logger LOG = Logger.getLogger(ProtocolHandler.class
       .getName());
 
   /**
    * Creates a new ProtocolHandler for the given Config object
-   * 
+   *
    * @param config
    *          The Config object that guides this ProtocolHandler in making class
    *          instanciations
@@ -87,7 +87,7 @@ public class ProtocolHandler {
 
   /**
    * Returns the appropriate protocol for the given Path
-   * 
+   *
    * @param ProtocolPath
    *          Used to determine the appropriate Protocol to be returned and the
    *          path to navigate on if navigateToPathLoc is set to true.
@@ -113,7 +113,7 @@ public class ProtocolHandler {
         if (pFile.isDir())
           this.cd(protocol, pFile);
         else
-          this.cd(protocol, (RemoteSiteFile) pFile.getParent());
+          this.cd(protocol, new RemoteSiteFile(pFile.getParent(), pFile.getSite()));
       }
       return protocol;
     } catch (Exception e) {
@@ -174,9 +174,9 @@ public class ProtocolHandler {
     return protocol;
   }
 
-  public synchronized List<RemoteSiteFile> nextPage(Protocol protocol)
+  public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol)
       throws RemoteConnectionException, ProtocolException {
-    return nextPage(protocol, null);
+    return nextPage(site, protocol, null);
   }
 
   /**
@@ -185,25 +185,25 @@ public class ProtocolHandler {
    * @throws RemoteConnectionException
    * @throws ProtocolException
    */
-  public synchronized List<RemoteSiteFile> nextPage(Protocol protocol,
+  public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol,
       ProtocolFileFilter filter) throws RemoteConnectionException,
       ProtocolException {
 
-    PagingInfo pgInfo = this.getPagingInfo(this.pwd(protocol));
+    PagingInfo pgInfo = this.getPagingInfo(this.pwd(site, protocol));
     try {
       System.out.println("PageSize: " + pi.getPageSize() + " PageLoc: "
           + pgInfo.getPageLoc());
-      List<RemoteSiteFile> fileList = this.ls(protocol);
+      List<RemoteSiteFile> fileList = this.ls(site, protocol);
       System.out.println("FileList size: " + fileList.size());
 
-      if (this.getDynamicFileList(protocol) == null
+      if (this.getDynamicFileList(site, protocol) == null
           && !this.passesDynamicDetection(pgInfo, fileList)) {
         LOG.log(
             Level.SEVERE,
             "Remote directory '"
-                + this.pwd(protocol)
+                + this.pwd(site, protocol)
                 + "' file list size has changed -- setting directory as dynamic and resetting
page location");
-        this.putDynamicFileList(protocol, fileList);
+        this.putDynamicFileList(site, protocol, fileList);
         pgInfo.updatePageInfo(0, fileList);
       }
 
@@ -271,13 +271,13 @@ public class ProtocolHandler {
     } catch (Exception e) {
       downloadFile.delete();
       throw new RemoteConnectionException("Failed to download file " + fromFile
-          + " : " + e.getMessage());
+          + " : " + e.getMessage(), e);
     }
   }
 
   /**
    * Connects the given Protocol to the given URL
-   * 
+   *
    * @param protocol
    *          The Protocol that will be connected
    * @param url
@@ -347,8 +347,8 @@ public class ProtocolHandler {
           + " . . . this may take a few minutes . . .");
       // test ls, cd, and pwd
       this.cdToHOME(protocol);
-      RemoteSiteFile home = this.pwd(protocol);
-      this.ls(protocol);
+      RemoteSiteFile home = this.pwd(remoteSite, protocol);
+      this.ls(remoteSite, protocol);
       if (remoteSite.getCdTestDir() != null)
         this.cd(protocol, new RemoteSiteFile(home, remoteSite.getCdTestDir(),
             true, remoteSite));
@@ -384,9 +384,9 @@ public class ProtocolHandler {
     protocol.cd(file);
   }
 
-  public RemoteSiteFile getProtocolFileFor(Protocol protocol, String file,
+  public RemoteSiteFile getProtocolFileFor(RemoteSite site, Protocol protocol, String file,
       boolean isDir) throws ProtocolException {
-    return this.getProtocolFileByProtocol(protocol, file, isDir);
+    return this.getProtocolFileByProtocol(site, protocol, file, isDir);
   }
 
   public synchronized boolean delete(Protocol protocol, RemoteSiteFile file)
@@ -426,50 +426,50 @@ public class ProtocolHandler {
     return pgInfo;
   }
 
-  public RemoteSiteFile pwd(Protocol protocol) throws ProtocolException {
-    return new RemoteSiteFile(protocol.pwd());
+  public RemoteSiteFile pwd(RemoteSite site, Protocol protocol) throws ProtocolException
{
+    return new RemoteSiteFile(protocol.pwd(), site);
   }
 
   public List<RemoteSiteFile> ls(Protocol protocol, RemoteSiteFile dir)
       throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+    List<RemoteSiteFile> fileList = this.getDynamicFileList(dir.getSite(), protocol);
     if (fileList == null) {
       protocol.cd(dir);
-      fileList = toRemoteSiteFiles(protocol.ls());
+      fileList = toRemoteSiteFiles(protocol.ls(), dir.getSite());
     }
     return fileList;
   }
 
-  public List<RemoteSiteFile> ls(Protocol protocol) throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+  public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol) throws ProtocolException
{
+    List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
     if (fileList == null)
-      fileList = toRemoteSiteFiles(protocol.ls());
+      fileList = toRemoteSiteFiles(protocol.ls(), site);
     return fileList;
   }
 
-  public List<RemoteSiteFile> ls(Protocol protocol, ProtocolFileFilter filter)
+  public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol, ProtocolFileFilter
filter)
       throws ProtocolException {
-    List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+    List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
     if (fileList == null)
-      fileList = toRemoteSiteFiles(protocol.ls(filter));
+      fileList = toRemoteSiteFiles(protocol.ls(filter), site);
     return fileList;
   }
 
-  private synchronized List<RemoteSiteFile> getDynamicFileList(Protocol protocol)
+  private synchronized List<RemoteSiteFile> getDynamicFileList(RemoteSite site, Protocol
protocol)
       throws ProtocolException {
     return (List<RemoteSiteFile>) (List<?>) this.pathAndFileListMap.get(this
-        .pwd(protocol));
+        .pwd(site, protocol));
   }
 
-  private synchronized void putDynamicFileList(Protocol protocol,
+  private synchronized void putDynamicFileList(RemoteSite site, Protocol protocol,
       List<RemoteSiteFile> fileList) throws ProtocolException {
-    this.pathAndFileListMap.put(this.pwd(protocol), fileList);
+    this.pathAndFileListMap.put(this.pwd(site, protocol), fileList);
   }
 
-  public synchronized RemoteSiteFile getHomeDir(Protocol protocol) {
+  public synchronized RemoteSiteFile getHomeDir(RemoteSite site, Protocol protocol) {
     try {
       protocol.cdHome();
-      return new RemoteSiteFile(protocol.pwd());
+      return new RemoteSiteFile(protocol.pwd(), site);
     } catch (Exception e) {
       e.printStackTrace();
       return null;
@@ -488,19 +488,17 @@ public class ProtocolHandler {
 
   /**
    * Disconnects and logs out the given Protocol
-   * 
+   *
    * @param protocol
    *          The Protocol to be logout out and disconnected
    * @throws RemoteConnectionException
    */
   public void disconnect(Protocol protocol) throws RemoteConnectionException {
-    URL url = null;
     try {
-      url = ((RemoteSiteFile) protocol.pwd()).getSite().getURL();
-      LOG.log(Level.INFO, "Disconnecting protocol from " + url);
+      LOG.log(Level.INFO, "Disconnecting protocol " + protocol.getClass().getName());
       protocol.close();
     } catch (Exception e) {
-      throw new RemoteConnectionException("Error disconnecting from " + url
+      throw new RemoteConnectionException("Error disconnecting " + protocol.getClass().getName()
           + " : " + e.getMessage());
     }
   }
@@ -508,7 +506,7 @@ public class ProtocolHandler {
   /**
    * Disconnects all waiting Protocols and clears the waiting lists. Also clears
    * the current Protocol
-   * 
+   *
    * @throws RemoteConnectionException
    */
   public void close() throws RemoteConnectionException {
@@ -523,24 +521,24 @@ public class ProtocolHandler {
   }
 
   private synchronized RemoteSiteFile getProtocolFileByProtocol(
-      Protocol protocol, String file, boolean isDir) throws ProtocolException {
+      RemoteSite site, Protocol protocol, String file, boolean isDir) throws ProtocolException
{
     try {
       if (!file.startsWith("/")) {
         protocol.cdHome();
         file = protocol.pwd().getPath() + "/" + file;
       }
-      return new RemoteSiteFile(file, isDir, null);
+      return new RemoteSiteFile(file, isDir, site);
     } catch (Exception e) {
       throw new ProtocolException("Failed to create protocol for " + file
           + " : " + e.getMessage());
     }
   }
 
-  private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files) {
+  private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files, RemoteSite
site) {
     List<RemoteSiteFile> newFiles = new Vector<RemoteSiteFile>();
     if (files != null) {
       for (ProtocolFile file : files) {
-        newFiles.add(new RemoteSiteFile(file));
+        newFiles.add(new RemoteSiteFile(file, site));
       }
     }
     return newFiles;

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
Fri Jul 27 19:05:29 2012
@@ -21,19 +21,19 @@ package org.apache.oodt.cas.pushpull.pro
 import org.apache.oodt.cas.protocol.ProtocolFile;
 
 /**
- * 
+ *
  * Extends {@link ProtocolFile} and links it to a {@link RemoteSite}.
- * 
+ *
  * @author mattmann
  * @version $Revision$
- * 
+ *
  */
 public class RemoteSiteFile extends ProtocolFile {
 
   private RemoteSite site;
-  
-  public RemoteSiteFile(ProtocolFile file){
-    this(file.getPath(), file.isDir(), null);
+
+  public RemoteSiteFile(ProtocolFile file, RemoteSite site){
+     this(file.getPath(), file.isDir(), site);
   }
 
   /**
@@ -78,7 +78,8 @@ public class RemoteSiteFile extends Prot
     return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
   }
 
-  public RemoteSiteFile getAbsoluteFile() {
+  @Override
+public RemoteSiteFile getAbsoluteFile() {
     ProtocolFile parent = super.getAbsoluteFile();
     return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
   }

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
Fri Jul 27 19:05:29 2012
@@ -49,10 +49,10 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
- * 
+ *
  * @author bfoster
  * @version $Revision$
- * 
+ *
  * <p>
  * Describe your class here
  * </p>.
@@ -65,12 +65,13 @@ public class RemoteCrawler implements Re
     /**
      * Starts the crawler and creates a default DirStruct if null was supplied
      * in constructor
-     * 
+     *
      * @throws MalformedURLException
      * @throws ProtocolException
      * @throws ProtocolFileException
      */
-    public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
+    @Override
+   public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
             File propFile, DataFilesInfo dfi, DataFileToPropFileLinker linker)
             throws Exception {
         RemoteSite remoteSite;
@@ -104,7 +105,7 @@ public class RemoteCrawler implements Re
 
         // add starting directory to stack
         Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
-        files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite)));
+        files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite), remoteSite));
 
         // start crawling
         while (!files.isEmpty()) {
@@ -116,8 +117,9 @@ public class RemoteCrawler implements Re
                     // get next page worth of children
                     List<RemoteSiteFile> children = frs.getNextPage(file,
                             new ProtocolFileFilter() {
-                                public boolean accept(ProtocolFile pFile) {
-                                    return FileRestrictions.isAllowed(new 
+                                @Override
+                              public boolean accept(ProtocolFile pFile) {
+                                    return FileRestrictions.isAllowed(new
                                         ProtocolPath(pFile
                                             .getPath(), pFile.isDir()), vf);
                                 }
@@ -156,7 +158,7 @@ public class RemoteCrawler implements Re
                 linker.markAsFailed(propFile, e.getMessage());
                 throw new Exception("Uknown error accured while downloading "
                         + file + " from " + remoteSite + " -- bailing out : "
-                        + e.getMessage());
+                        + e.getMessage(), e);
             }
         }
     }

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
Fri Jul 27 19:05:29 2012
@@ -54,38 +54,40 @@ import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import com.google.common.base.Preconditions;
+
 /**
  * <pre>
- *    Will crawl external directory structures and will download the files within these structures.

- *    
+ *    Will crawl external directory structures and will download the files within these structures.
+ *
  *    This class's settings are set using a java .properties file which can be read in and
parsed by Config.java.
  *    This .properties file should have the following properties set:
- *     
+ *
  *   	{@literal #list of sites to crawl
  *   	protocol.external.sources=&lt;path-to-xml-file&gt;
- *   
+ *
  *   	#protocol types
  *   	protocolfactory.types=&lt;list-of-protocols-separated-by-commas&gt; (e.g. ftp,http,https,sftp)
- *   
+ *
  *   	#Protocol factories per types (must have one for each protocol mention in protocolfactory.types
-- the property must be name
  *    	# as such: protocolfactory.&lt;name-of-protocol-type&gt;
  *   	protocolfactory.ftp=&lt;path-to-java-protocolfactory-class&gt; (e.g. org.apache.oodt.cas.protocol.ftp.FtpClientFactory)
  *   	protocolfactory.http=&lt;path-to-java-protocolfactory-class&gt;
  *   	protocolfactory.https=&lt;path-to-java-protocolfactory-class&gt;
  *   	protocolfactory.sftp=&lt;path-to-java-protocolfactory-class&gt;
- *   
+ *
  *   	#configuration to make java.net.URL accept unsupported protocols -- must exist just
as shown
  *   	java.protocol.handler.pkgs=org.apache.oodt.cas.url.handlers
  *    }
- *    
- *    In order to specify which external sites to crawl you must create a XML file which
contains the 
+ *
+ *    In order to specify which external sites to crawl you must create a XML file which
contains the
  *    the site and necessary information needed to crawl the site, such as username and password.
  *    protocol.external.sources must contain the path to this file so the crawl knows where
to find it.
  *    You can also train this class on how to crawl each given site.  This is also specified
in an XML
  *    file, whose path must be given in the first mentioned XML file which contians the username
and password.
- *    
+ *
  *    Then schema for the external sites XML file is as such:
- *    
+ *
  *    	{@literal &lt;sources&gt;
  *    	   &lt;source url=&quot;url-of-server&quot;&gt;
  *    	      &lt;username&gt;username&lt;/username&gt;
@@ -97,7 +99,7 @@ import java.util.logging.Logger;
  *    	   ...
  *    	   ...
  *    	&lt;/sources\&gt;}
- *    
+ *
  *    You may specify as many sources as you would like by specifying multiple {@literal
&lt;source&gt;} tags.
  *    In the {@literal &lt;source&gt;} tag, the parameter 'url' must be specified.
 This is the url of the server
  *    you want the crawler to connect to.  It should be of the following format:
@@ -110,7 +112,7 @@ import java.util.logging.Logger;
  *    element is optional.  If no {@literal &lt;dirStruct&gt;} is given, then every
directory will be crawled on the site
  *    and every encountered file will be downloaded.
  * </pre>
- * 
+ *
  * @author bfoster
  */
 public class FileRetrievalSystem {
@@ -132,7 +134,7 @@ public class FileRetrievalSystem {
      */
     private int max_sessions;
 
-    private int absMaxAllowedSessions = 50;
+    private final int absMaxAllowedSessions = 50;
 
     /**
      * This is just for clarity purposes. . .I only create the amount of threads
@@ -181,7 +183,7 @@ public class FileRetrievalSystem {
     /**
      * Creates a Crawler based on the URL, DirStruct, and Config objects passed
      * in. If no DirStruct is needed then set it to null.
-     * 
+     *
      * @param url
      *            The URL for which you want this Crawler to crawl
      * @param dirStruct
@@ -223,7 +225,7 @@ public class FileRetrievalSystem {
 
     /**
      * Initializes variables that must be reset when more than one crawl is done
-     * 
+     *
      * @throws ThreadEvaluatorException
      */
     void resetVariables() throws ThreadEvaluatorException {
@@ -258,10 +260,11 @@ public class FileRetrievalSystem {
             final ProtocolFileFilter filter) throws RemoteConnectionException {
         for (int i = 0; i < 3; i++) {
             try {
-                return protocolHandler.nextPage(protocolHandler
+                return protocolHandler.nextPage(dir.getSite(), protocolHandler
                         .getAppropriateProtocol(dir, true, true),
                         new ProtocolFileFilter() {
-                            public boolean accept(ProtocolFile file) {
+                            @Override
+                           public boolean accept(ProtocolFile file) {
                                 return filter.accept(file)
                                         && !FileRetrievalSystem.this
                                                 .isDownloading(file);
@@ -269,7 +272,7 @@ public class FileRetrievalSystem {
                         });
             } catch (Exception e) {
                 LOG.log(Level.WARNING, "Retrying to get next page for " + dir
-                        + " because operation failed : " + e.getMessage());
+                        + " because operation failed : " + e.getMessage(), e);
             }
         }
         throw new RemoteConnectionException("Failed to get next page for "
@@ -298,7 +301,7 @@ public class FileRetrievalSystem {
             throws MalformedURLException, ProtocolException {
         if (validate(remoteSite))
             this
-                    .changeToDir(protocolHandler.getProtocolFileFor(
+                    .changeToDir(protocolHandler.getProtocolFileFor(remoteSite,
                             protocolHandler.getAppropriateProtocolBySite(
                                     remoteSite, true), dir, true));
         else
@@ -318,7 +321,7 @@ public class FileRetrievalSystem {
     public ProtocolFile getHomeDir(RemoteSite remoteSite)
             throws ProtocolException {
         if (validate(remoteSite))
-            return protocolHandler.getHomeDir(protocolHandler
+            return protocolHandler.getHomeDir(remoteSite, protocolHandler
                     .getAppropriateProtocolBySite(remoteSite, true));
         else
             throw new ProtocolException("Not a valid remote site " + remoteSite);
@@ -327,7 +330,7 @@ public class FileRetrievalSystem {
     public ProtocolFile getProtocolFile(RemoteSite remoteSite, String file,
             boolean isDir) throws ProtocolException {
         if (validate(remoteSite))
-            return protocolHandler.getProtocolFileFor(protocolHandler
+            return protocolHandler.getProtocolFileFor(remoteSite, protocolHandler
                     .getAppropriateProtocolBySite(remoteSite, true), file,
                     isDir);
         else
@@ -338,7 +341,7 @@ public class FileRetrievalSystem {
             throws ProtocolFileException, ProtocolException,
             MalformedURLException {
         if (validate(remoteSite))
-            return protocolHandler.pwd(protocolHandler
+            return protocolHandler.pwd(remoteSite, protocolHandler
                     .getAppropriateProtocolBySite(remoteSite, true));
         else
             throw new ProtocolException("Not a valid remote site " + remoteSite);
@@ -355,7 +358,7 @@ public class FileRetrievalSystem {
         if (validate(remoteSite)) {
             if (!file.startsWith("/"))
                 file = "/" + file;
-            return addToDownloadQueue(protocolHandler.getProtocolFileFor(
+            return addToDownloadQueue(protocolHandler.getProtocolFileFor(remoteSite,
                     protocolHandler.getAppropriateProtocolBySite(remoteSite,
                             true), file, false), renamingString, downloadToDir,
                     uniqueMetadataElement, deleteAfterDownload);
@@ -364,6 +367,7 @@ public class FileRetrievalSystem {
     }
 
     public boolean validate(RemoteSite remoteSite) {
+        Preconditions.checkNotNull(remoteSite);
         LinkedList<RemoteSite> remoteSites = this.siteInfo
                 .getPossibleRemoteSites(remoteSite.getAlias(), remoteSite
                         .getURL(), remoteSite.getUsername(), remoteSite
@@ -409,12 +413,12 @@ public class FileRetrievalSystem {
                     + "' because it is already on the download queue");
             return false;
         }
-        
+
         RemoteFile remoteFile = new RemoteFile(file);
         remoteFile.addMetadata(RemoteFile.RENAMING_STRING, renamingString);
         remoteFile.addMetadata(RemoteFile.DELETE_AFTER_DOWNLOAD,
                 deleteAfterDownload + "");
-        
+
         String mimeType = this.mimeTypeDetection.getMimeType(file.getName());
         if (mimeType != null
                 && !mimeType.equals("application/octet-stream")) {
@@ -428,7 +432,7 @@ public class FileRetrievalSystem {
                 for (String field : description.split("\\&\\&")) {
                   String[] keyval = field.split("\\=");
                   remoteFile.addMetadata(keyval[0].trim(), keyval[1].trim());
-                }                   
+                }
               }
               else{
                 // it's the ProductType
@@ -447,15 +451,15 @@ public class FileRetrievalSystem {
                 + downloadToDir.getPath());
         if (!this.isStagingAreaInitialized(downloadToDir))
             this.initializeStagingArea(downloadToDir);
-        
+
         remoteFile.addMetadata(RemoteFile.DOWNLOAD_TO_DIR, downloadToDir.getAbsolutePath());
-        
+
     	if (remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR) != null) {
     		remoteFile.addMetadata(RemoteFile.PRODUCT_NAME, RenamingConvention.rename(remoteFile.getProtocolFile(),
remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR)));
     	}else {
     		remoteFile.setUniqueMetadataElement(uniqueMetadataElement == null ? RemoteFile.FILENAME
: uniqueMetadataElement);
     	}
-        
+
         if (!isAlreadyInDatabase(remoteFile)) {
 
             // get download location
@@ -522,7 +526,8 @@ public class FileRetrievalSystem {
         LOG.log(Level.INFO, "Preparing staging area " + stagingArea);
         if (stagingArea.exists()) {
             File[] failedDownloads = stagingArea.listFiles(new FileFilter() {
-                public boolean accept(File pathname) {
+                @Override
+               public boolean accept(File pathname) {
                     return pathname.getName().startsWith("Downloading_");
                 }
             });
@@ -574,7 +579,7 @@ public class FileRetrievalSystem {
 
     /**
      * Sleeps the crawling thread
-     * 
+     *
      * @throws InterruptedException
      */
     synchronized void waitMainThread() throws InterruptedException {
@@ -602,7 +607,7 @@ public class FileRetrievalSystem {
     /**
      * Gets an available downloading session Protocol. Returns null if none are
      * available
-     * 
+     *
      * @param path
      *            The session returned will be checked against the Path passed
      *            in and if not presently connected to the Path's URL, it will
@@ -652,7 +657,8 @@ public class FileRetrievalSystem {
                 if (file.isDir())
                     protocolHandler.cd(session, file);
                 else
-                    protocolHandler.cd(session, new RemoteSiteFile(file.getParent()));
+                    protocolHandler.cd(session,
+                          new RemoteSiteFile(file.getParent(), file.getSite()));
             } catch (Exception e) {
                 e.printStackTrace();
                 try {
@@ -668,7 +674,7 @@ public class FileRetrievalSystem {
 
     /**
      * Puts a session in the available session list
-     * 
+     *
      * @param session
      *            The Protocol session to be added to the available list
      */
@@ -678,7 +684,7 @@ public class FileRetrievalSystem {
 
     /**
      * Removes a session from the available list and returns it
-     * 
+     *
      * @return An available downloading Protocol session
      */
     synchronized Protocol getAvailableSession() {
@@ -692,7 +698,7 @@ public class FileRetrievalSystem {
     /**
      * Registers a downloading session with the threadpoolexecutor to begin
      * downloading the specified ProtocolFile to the local File location
-     * 
+     *
      * @param session
      *            The downloading Protocol session to be used to download the
      *            ProtocolFile
@@ -705,6 +711,7 @@ public class FileRetrievalSystem {
             final RemoteFile remoteFile, final File newFile) {
         this.addToDownloadingList(remoteFile.getProtocolFile());
         threadController.execute(new Runnable() {
+            @Override
             public void run() {
                 boolean successful = false;
                 int retries = 0;
@@ -797,7 +804,7 @@ public class FileRetrievalSystem {
                             LOG.log(Level.WARNING, "Retrying to download file "
                                     + remoteFile.getProtocolFile()
                                     + " because download failed : "
-                                    + e.getMessage());
+                                    + e.getMessage(), e);
                             try {
                                 protocolHandler.disconnect(curSession);
                             } catch (Exception exc) {
@@ -811,7 +818,7 @@ public class FileRetrievalSystem {
                                         "Failed to reconnect protocol to retry download of
file "
                                                 + remoteFile.getProtocolFile()
                                                 + " -- aborting retry : "
-                                                + e.getMessage());
+                                                + e.getMessage(), e);
                             }
                         } else {
                             LOG
@@ -821,7 +828,7 @@ public class FileRetrievalSystem {
                                                     + remoteFile
                                                             .getProtocolFile()
                                                     + " do to too many previous download
failures : "
-                                                    + e.getMessage());
+                                                    + e.getMessage(), e);
                             if (FileRetrievalSystem.this.dListener != null)
                                 FileRetrievalSystem.this.dListener
                                         .downloadFailed(remoteFile
@@ -924,7 +931,7 @@ public class FileRetrievalSystem {
      * list. The ThreadPoolExecutor needs to be completely shutdown before this
      * method should be called. Otherwise some Protocols might not be
      * disconnected or left downloading.
-     * 
+     *
      * @return True if successful, false otherwise
      * @throws RemoteConnectionException
      */

Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
Fri Jul 27 19:05:29 2012
@@ -44,27 +44,27 @@ import org.apache.oodt.cas.pushpull.retr
 import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
 
 /**
- * 
+ *
  * @author bfoster
  * @version $Revision$
- * 
+ *
  * <p>
  * Describe your class here
  * </p>.
  */
 public class RetrievalSetup {
 
-    private Config config;
+    private final Config config;
 
-    private HashSet<File> alreadyProcessedPropFiles;
+    private final HashSet<File> alreadyProcessedPropFiles;
 
-    private HashMap<Class<RetrievalMethod>, RetrievalMethod> classToRmMap;
+    private final HashMap<Class<RetrievalMethod>, RetrievalMethod> classToRmMap;
 
     private boolean downloadingProps;
 
-    private SiteInfo siteInfo;
+    private final SiteInfo siteInfo;
 
-    private DataFileToPropFileLinker linker;
+    private final DataFileToPropFileLinker linker;
 
     private final static Logger LOG = Logger.getLogger(RetrievalSetup.class
             .getName());
@@ -110,7 +110,7 @@ public class RetrievalSetup {
                                 linker);
                     } catch (ParserException e) {
                         LOG.log(Level.SEVERE, "Failed to parse property file "
-                                + propFile + " : " + e.getMessage());
+                                + propFile + " : " + e.getMessage(), e);
                         linker.markAsFailed(propFile,
                                 "Failed to parse property file " + propFile
                                         + " : " + e.getMessage());
@@ -118,7 +118,7 @@ public class RetrievalSetup {
                         LOG.log(Level.SEVERE,
                                 "Failed to finish downloading per property files "
                                         + propFile.getAbsolutePath() + " : "
-                                        + e.getMessage());
+                                        + e.getMessage(), e);
                         linker.markAsFailed(propFile,
                                 "Error while downloading per property file "
                                         + propFile.getAbsolutePath() + " : "
@@ -160,7 +160,8 @@ public class RetrievalSetup {
             this.downloadingProps = true;
             new Thread(new Runnable() {
 
-                public void run() {
+                @Override
+               public void run() {
                     FileRetrievalSystem frs = null;
                     try {
                         (frs = new FileRetrievalSystem(
@@ -217,6 +218,7 @@ public class RetrievalSetup {
 
     private File[] getCurrentlyDownloadedPropFiles(final PropFilesInfo pfi) {
         File[] files = pfi.getLocalDir().listFiles(new FileFilter() {
+            @Override
             public boolean accept(File pathname) {
                 return pfi.getParserForFile(pathname) != null
                         && !(pathname.getName().startsWith("Downloading_")
@@ -247,7 +249,7 @@ public class RetrievalSetup {
         File newLoc = new File(moveToDir, dirstructFile.getName());
         dirstructFile.renameTo(newLoc);
         new File(dirstructFile.getAbsolutePath() + "." + config.getMetFileExtension())
-                .renameTo(new File(newLoc.getAbsolutePath() + "." 
+                .renameTo(new File(newLoc.getAbsolutePath() + "."
                 		+ config.getMetFileExtension()));
         if (errorMsgs != null) {
             File errorFile = new File(newLoc.getParentFile(), dirstructFile



Mime
View raw message