Author: bfoster
Date: Fri Jul 27 19:05:29 2012
New Revision: 1366499
URL: http://svn.apache.org/viewvc?rev=1366499&view=rev
Log:
- Fixed: RemoteSiteFile doesn't always set RemoteSite
---------------
OODT-476
Modified:
oodt/trunk/pushpull/pom.xml
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
Modified: oodt/trunk/pushpull/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/pom.xml?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/pom.xml (original)
+++ oodt/trunk/pushpull/pom.xml Fri Jul 27 19:05:29 2012
@@ -147,6 +147,11 @@
<version>2.0.8</version>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>10.0.1</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.2</version>
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/PushPullFrameworkException.java
Fri Jul 27 19:05:29 2012
@@ -19,10 +19,10 @@
package org.apache.oodt.cas.pushpull.exceptions;
/**
- *
+ *
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* Describe your class here
* </p>.
@@ -39,6 +39,10 @@ public class PushPullFrameworkException
super(PushPullFrameworkException.addCallingClassToMsg(msg));
}
+ public PushPullFrameworkException(String msg, Throwable t) {
+ super(PushPullFrameworkException.addCallingClassToMsg(msg), t);
+ }
+
private static String addCallingClassToMsg(String msg) {
try {
Throwable stack = new Throwable();
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/exceptions/RemoteConnectionException.java
Fri Jul 27 19:05:29 2012
@@ -19,10 +19,10 @@
package org.apache.oodt.cas.pushpull.exceptions;
/**
- *
+ *
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* Describe your class here
* </p>.
@@ -38,4 +38,8 @@ public class RemoteConnectionException e
public RemoteConnectionException(String msg) {
super(msg);
}
+
+ public RemoteConnectionException(String msg, Throwable t) {
+ super(msg, t);
+ }
}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/ProtocolHandler.java
Fri Jul 27 19:05:29 2012
@@ -52,27 +52,27 @@ import java.util.logging.Logger;
* 'allowReuse' to false then a new Protocol object will be created and
* returned.<br>
* <br>
- *
+ *
* @author bfoster
*/
public class ProtocolHandler {
- private HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
+ private final HashMap<URL, ProtocolFactory> urlAndProtocolFactory;
- private HashMap<URL, Protocol> reuseProtocols;
+ private final HashMap<URL, Protocol> reuseProtocols;
- private HashMap<RemoteSiteFile, PagingInfo> pageInfos;
+ private final HashMap<RemoteSiteFile, PagingInfo> pageInfos;
- private HashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
+ private final HashMap<RemoteSiteFile, List<RemoteSiteFile>> pathAndFileListMap;
- private ProtocolInfo pi;
+ private final ProtocolInfo pi;
private static final Logger LOG = Logger.getLogger(ProtocolHandler.class
.getName());
/**
* Creates a new ProtocolHandler for the given Config object
- *
+ *
* @param config
* The Config object that guides this ProtocolHandler in making class
* instanciations
@@ -87,7 +87,7 @@ public class ProtocolHandler {
/**
* Returns the appropriate protocol for the given Path
- *
+ *
* @param ProtocolPath
* Used to determine the appropriate Protocol to be returned and the
* path to navigate on if navigateToPathLoc is set to true.
@@ -113,7 +113,7 @@ public class ProtocolHandler {
if (pFile.isDir())
this.cd(protocol, pFile);
else
- this.cd(protocol, (RemoteSiteFile) pFile.getParent());
+ this.cd(protocol, new RemoteSiteFile(pFile.getParent(), pFile.getSite()));
}
return protocol;
} catch (Exception e) {
@@ -174,9 +174,9 @@ public class ProtocolHandler {
return protocol;
}
- public synchronized List<RemoteSiteFile> nextPage(Protocol protocol)
+ public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol)
throws RemoteConnectionException, ProtocolException {
- return nextPage(protocol, null);
+ return nextPage(site, protocol, null);
}
/**
@@ -185,25 +185,25 @@ public class ProtocolHandler {
* @throws RemoteConnectionException
* @throws ProtocolException
*/
- public synchronized List<RemoteSiteFile> nextPage(Protocol protocol,
+ public synchronized List<RemoteSiteFile> nextPage(RemoteSite site, Protocol protocol,
ProtocolFileFilter filter) throws RemoteConnectionException,
ProtocolException {
- PagingInfo pgInfo = this.getPagingInfo(this.pwd(protocol));
+ PagingInfo pgInfo = this.getPagingInfo(this.pwd(site, protocol));
try {
System.out.println("PageSize: " + pi.getPageSize() + " PageLoc: "
+ pgInfo.getPageLoc());
- List<RemoteSiteFile> fileList = this.ls(protocol);
+ List<RemoteSiteFile> fileList = this.ls(site, protocol);
System.out.println("FileList size: " + fileList.size());
- if (this.getDynamicFileList(protocol) == null
+ if (this.getDynamicFileList(site, protocol) == null
&& !this.passesDynamicDetection(pgInfo, fileList)) {
LOG.log(
Level.SEVERE,
"Remote directory '"
- + this.pwd(protocol)
+ + this.pwd(site, protocol)
+ "' file list size has changed -- setting directory as dynamic and resetting
page location");
- this.putDynamicFileList(protocol, fileList);
+ this.putDynamicFileList(site, protocol, fileList);
pgInfo.updatePageInfo(0, fileList);
}
@@ -271,13 +271,13 @@ public class ProtocolHandler {
} catch (Exception e) {
downloadFile.delete();
throw new RemoteConnectionException("Failed to download file " + fromFile
- + " : " + e.getMessage());
+ + " : " + e.getMessage(), e);
}
}
/**
* Connects the given Protocol to the given URL
- *
+ *
* @param protocol
* The Protocol that will be connected
* @param url
@@ -347,8 +347,8 @@ public class ProtocolHandler {
+ " . . . this may take a few minutes . . .");
// test ls, cd, and pwd
this.cdToHOME(protocol);
- RemoteSiteFile home = this.pwd(protocol);
- this.ls(protocol);
+ RemoteSiteFile home = this.pwd(remoteSite, protocol);
+ this.ls(remoteSite, protocol);
if (remoteSite.getCdTestDir() != null)
this.cd(protocol, new RemoteSiteFile(home, remoteSite.getCdTestDir(),
true, remoteSite));
@@ -384,9 +384,9 @@ public class ProtocolHandler {
protocol.cd(file);
}
- public RemoteSiteFile getProtocolFileFor(Protocol protocol, String file,
+ public RemoteSiteFile getProtocolFileFor(RemoteSite site, Protocol protocol, String file,
boolean isDir) throws ProtocolException {
- return this.getProtocolFileByProtocol(protocol, file, isDir);
+ return this.getProtocolFileByProtocol(site, protocol, file, isDir);
}
public synchronized boolean delete(Protocol protocol, RemoteSiteFile file)
@@ -426,50 +426,50 @@ public class ProtocolHandler {
return pgInfo;
}
- public RemoteSiteFile pwd(Protocol protocol) throws ProtocolException {
- return new RemoteSiteFile(protocol.pwd());
+ public RemoteSiteFile pwd(RemoteSite site, Protocol protocol) throws ProtocolException
{
+ return new RemoteSiteFile(protocol.pwd(), site);
}
public List<RemoteSiteFile> ls(Protocol protocol, RemoteSiteFile dir)
throws ProtocolException {
- List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(dir.getSite(), protocol);
if (fileList == null) {
protocol.cd(dir);
- fileList = toRemoteSiteFiles(protocol.ls());
+ fileList = toRemoteSiteFiles(protocol.ls(), dir.getSite());
}
return fileList;
}
- public List<RemoteSiteFile> ls(Protocol protocol) throws ProtocolException {
- List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol) throws ProtocolException
{
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
if (fileList == null)
- fileList = toRemoteSiteFiles(protocol.ls());
+ fileList = toRemoteSiteFiles(protocol.ls(), site);
return fileList;
}
- public List<RemoteSiteFile> ls(Protocol protocol, ProtocolFileFilter filter)
+ public List<RemoteSiteFile> ls(RemoteSite site, Protocol protocol, ProtocolFileFilter
filter)
throws ProtocolException {
- List<RemoteSiteFile> fileList = this.getDynamicFileList(protocol);
+ List<RemoteSiteFile> fileList = this.getDynamicFileList(site, protocol);
if (fileList == null)
- fileList = toRemoteSiteFiles(protocol.ls(filter));
+ fileList = toRemoteSiteFiles(protocol.ls(filter), site);
return fileList;
}
- private synchronized List<RemoteSiteFile> getDynamicFileList(Protocol protocol)
+ private synchronized List<RemoteSiteFile> getDynamicFileList(RemoteSite site, Protocol
protocol)
throws ProtocolException {
return (List<RemoteSiteFile>) (List<?>) this.pathAndFileListMap.get(this
- .pwd(protocol));
+ .pwd(site, protocol));
}
- private synchronized void putDynamicFileList(Protocol protocol,
+ private synchronized void putDynamicFileList(RemoteSite site, Protocol protocol,
List<RemoteSiteFile> fileList) throws ProtocolException {
- this.pathAndFileListMap.put(this.pwd(protocol), fileList);
+ this.pathAndFileListMap.put(this.pwd(site, protocol), fileList);
}
- public synchronized RemoteSiteFile getHomeDir(Protocol protocol) {
+ public synchronized RemoteSiteFile getHomeDir(RemoteSite site, Protocol protocol) {
try {
protocol.cdHome();
- return new RemoteSiteFile(protocol.pwd());
+ return new RemoteSiteFile(protocol.pwd(), site);
} catch (Exception e) {
e.printStackTrace();
return null;
@@ -488,19 +488,17 @@ public class ProtocolHandler {
/**
* Disconnects and logs out the given Protocol
- *
+ *
* @param protocol
* The Protocol to be logout out and disconnected
* @throws RemoteConnectionException
*/
public void disconnect(Protocol protocol) throws RemoteConnectionException {
- URL url = null;
try {
- url = ((RemoteSiteFile) protocol.pwd()).getSite().getURL();
- LOG.log(Level.INFO, "Disconnecting protocol from " + url);
+ LOG.log(Level.INFO, "Disconnecting protocol " + protocol.getClass().getName());
protocol.close();
} catch (Exception e) {
- throw new RemoteConnectionException("Error disconnecting from " + url
+ throw new RemoteConnectionException("Error disconnecting " + protocol.getClass().getName()
+ " : " + e.getMessage());
}
}
@@ -508,7 +506,7 @@ public class ProtocolHandler {
/**
* Disconnects all waiting Protocols and clears the waiting lists. Also clears
* the current Protocol
- *
+ *
* @throws RemoteConnectionException
*/
public void close() throws RemoteConnectionException {
@@ -523,24 +521,24 @@ public class ProtocolHandler {
}
private synchronized RemoteSiteFile getProtocolFileByProtocol(
- Protocol protocol, String file, boolean isDir) throws ProtocolException {
+ RemoteSite site, Protocol protocol, String file, boolean isDir) throws ProtocolException
{
try {
if (!file.startsWith("/")) {
protocol.cdHome();
file = protocol.pwd().getPath() + "/" + file;
}
- return new RemoteSiteFile(file, isDir, null);
+ return new RemoteSiteFile(file, isDir, site);
} catch (Exception e) {
throw new ProtocolException("Failed to create protocol for " + file
+ " : " + e.getMessage());
}
}
- private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files) {
+ private List<RemoteSiteFile> toRemoteSiteFiles(List<ProtocolFile> files, RemoteSite
site) {
List<RemoteSiteFile> newFiles = new Vector<RemoteSiteFile>();
if (files != null) {
for (ProtocolFile file : files) {
- newFiles.add(new RemoteSiteFile(file));
+ newFiles.add(new RemoteSiteFile(file, site));
}
}
return newFiles;
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/protocol/RemoteSiteFile.java
Fri Jul 27 19:05:29 2012
@@ -21,19 +21,19 @@ package org.apache.oodt.cas.pushpull.pro
import org.apache.oodt.cas.protocol.ProtocolFile;
/**
- *
+ *
* Extends {@link ProtocolFile} and links it to a {@link RemoteSite}.
- *
+ *
* @author mattmann
* @version $Revision$
- *
+ *
*/
public class RemoteSiteFile extends ProtocolFile {
private RemoteSite site;
-
- public RemoteSiteFile(ProtocolFile file){
- this(file.getPath(), file.isDir(), null);
+
+ public RemoteSiteFile(ProtocolFile file, RemoteSite site){
+ this(file.getPath(), file.isDir(), site);
}
/**
@@ -78,7 +78,8 @@ public class RemoteSiteFile extends Prot
return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
}
- public RemoteSiteFile getAbsoluteFile() {
+ @Override
+public RemoteSiteFile getAbsoluteFile() {
ProtocolFile parent = super.getAbsoluteFile();
return new RemoteSiteFile(parent.getPath(), parent.isDir(), this.site);
}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalmethod/RemoteCrawler.java
Fri Jul 27 19:05:29 2012
@@ -49,10 +49,10 @@ import java.util.logging.Level;
import java.util.logging.Logger;
/**
- *
+ *
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* Describe your class here
* </p>.
@@ -65,12 +65,13 @@ public class RemoteCrawler implements Re
/**
* Starts the crawler and creates a default DirStruct if null was supplied
* in constructor
- *
+ *
* @throws MalformedURLException
* @throws ProtocolException
* @throws ProtocolFileException
*/
- public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
+ @Override
+ public void processPropFile(FileRetrievalSystem frs, Parser propFileParser,
File propFile, DataFilesInfo dfi, DataFileToPropFileLinker linker)
throws Exception {
RemoteSite remoteSite;
@@ -104,7 +105,7 @@ public class RemoteCrawler implements Re
// add starting directory to stack
Stack<RemoteSiteFile> files = new Stack<RemoteSiteFile>();
- files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite)));
+ files.add(new RemoteSiteFile(frs.getCurrentFile(remoteSite), remoteSite));
// start crawling
while (!files.isEmpty()) {
@@ -116,8 +117,9 @@ public class RemoteCrawler implements Re
// get next page worth of children
List<RemoteSiteFile> children = frs.getNextPage(file,
new ProtocolFileFilter() {
- public boolean accept(ProtocolFile pFile) {
- return FileRestrictions.isAllowed(new
+ @Override
+ public boolean accept(ProtocolFile pFile) {
+ return FileRestrictions.isAllowed(new
ProtocolPath(pFile
.getPath(), pFile.isDir()), vf);
}
@@ -156,7 +158,7 @@ public class RemoteCrawler implements Re
linker.markAsFailed(propFile, e.getMessage());
throw new Exception("Uknown error accured while downloading "
+ file + " from " + remoteSite + " -- bailing out : "
- + e.getMessage());
+ + e.getMessage(), e);
}
}
}
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/FileRetrievalSystem.java
Fri Jul 27 19:05:29 2012
@@ -54,38 +54,40 @@ import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
+import com.google.common.base.Preconditions;
+
/**
* <pre>
- * Will crawl external directory structures and will download the files within these structures.
- *
+ * Will crawl external directory structures and will download the files within these structures.
+ *
* This class's settings are set using a java .properties file which can be read in and
parsed by Config.java.
* This .properties file should have the following properties set:
- *
+ *
* {@literal #list of sites to crawl
* protocol.external.sources=<path-to-xml-file>
- *
+ *
* #protocol types
* protocolfactory.types=<list-of-protocols-separated-by-commas> (e.g. ftp,http,https,sftp)
- *
+ *
* #Protocol factories per types (must have one for each protocol mention in protocolfactory.types
-- the property must be name
* # as such: protocolfactory.<name-of-protocol-type>
* protocolfactory.ftp=<path-to-java-protocolfactory-class> (e.g. org.apache.oodt.cas.protocol.ftp.FtpClientFactory)
* protocolfactory.http=<path-to-java-protocolfactory-class>
* protocolfactory.https=<path-to-java-protocolfactory-class>
* protocolfactory.sftp=<path-to-java-protocolfactory-class>
- *
+ *
* #configuration to make java.net.URL accept unsupported protocols -- must exist just
as shown
* java.protocol.handler.pkgs=org.apache.oodt.cas.url.handlers
* }
- *
- * In order to specify which external sites to crawl you must create a XML file which
contains the
+ *
+ * In order to specify which external sites to crawl you must create a XML file which
contains the
* the site and necessary information needed to crawl the site, such as username and password.
* protocol.external.sources must contain the path to this file so the crawl knows where
to find it.
* You can also train this class on how to crawl each given site. This is also specified
in an XML
* file, whose path must be given in the first mentioned XML file which contians the username
and password.
- *
+ *
* Then schema for the external sites XML file is as such:
- *
+ *
* {@literal <sources>
* <source url="url-of-server">
* <username>username</username>
@@ -97,7 +99,7 @@ import java.util.logging.Logger;
* ...
* ...
* </sources\>}
- *
+ *
* You may specify as many sources as you would like by specifying multiple {@literal
<source>} tags.
* In the {@literal <source>} tag, the parameter 'url' must be specified.
This is the url of the server
* you want the crawler to connect to. It should be of the following format:
@@ -110,7 +112,7 @@ import java.util.logging.Logger;
* element is optional. If no {@literal <dirStruct>} is given, then every
directory will be crawled on the site
* and every encountered file will be downloaded.
* </pre>
- *
+ *
* @author bfoster
*/
public class FileRetrievalSystem {
@@ -132,7 +134,7 @@ public class FileRetrievalSystem {
*/
private int max_sessions;
- private int absMaxAllowedSessions = 50;
+ private final int absMaxAllowedSessions = 50;
/**
* This is just for clarity purposes. . .I only create the amount of threads
@@ -181,7 +183,7 @@ public class FileRetrievalSystem {
/**
* Creates a Crawler based on the URL, DirStruct, and Config objects passed
* in. If no DirStruct is needed then set it to null.
- *
+ *
* @param url
* The URL for which you want this Crawler to crawl
* @param dirStruct
@@ -223,7 +225,7 @@ public class FileRetrievalSystem {
/**
* Initializes variables that must be reset when more than one crawl is done
- *
+ *
* @throws ThreadEvaluatorException
*/
void resetVariables() throws ThreadEvaluatorException {
@@ -258,10 +260,11 @@ public class FileRetrievalSystem {
final ProtocolFileFilter filter) throws RemoteConnectionException {
for (int i = 0; i < 3; i++) {
try {
- return protocolHandler.nextPage(protocolHandler
+ return protocolHandler.nextPage(dir.getSite(), protocolHandler
.getAppropriateProtocol(dir, true, true),
new ProtocolFileFilter() {
- public boolean accept(ProtocolFile file) {
+ @Override
+ public boolean accept(ProtocolFile file) {
return filter.accept(file)
&& !FileRetrievalSystem.this
.isDownloading(file);
@@ -269,7 +272,7 @@ public class FileRetrievalSystem {
});
} catch (Exception e) {
LOG.log(Level.WARNING, "Retrying to get next page for " + dir
- + " because operation failed : " + e.getMessage());
+ + " because operation failed : " + e.getMessage(), e);
}
}
throw new RemoteConnectionException("Failed to get next page for "
@@ -298,7 +301,7 @@ public class FileRetrievalSystem {
throws MalformedURLException, ProtocolException {
if (validate(remoteSite))
this
- .changeToDir(protocolHandler.getProtocolFileFor(
+ .changeToDir(protocolHandler.getProtocolFileFor(remoteSite,
protocolHandler.getAppropriateProtocolBySite(
remoteSite, true), dir, true));
else
@@ -318,7 +321,7 @@ public class FileRetrievalSystem {
public ProtocolFile getHomeDir(RemoteSite remoteSite)
throws ProtocolException {
if (validate(remoteSite))
- return protocolHandler.getHomeDir(protocolHandler
+ return protocolHandler.getHomeDir(remoteSite, protocolHandler
.getAppropriateProtocolBySite(remoteSite, true));
else
throw new ProtocolException("Not a valid remote site " + remoteSite);
@@ -327,7 +330,7 @@ public class FileRetrievalSystem {
public ProtocolFile getProtocolFile(RemoteSite remoteSite, String file,
boolean isDir) throws ProtocolException {
if (validate(remoteSite))
- return protocolHandler.getProtocolFileFor(protocolHandler
+ return protocolHandler.getProtocolFileFor(remoteSite, protocolHandler
.getAppropriateProtocolBySite(remoteSite, true), file,
isDir);
else
@@ -338,7 +341,7 @@ public class FileRetrievalSystem {
throws ProtocolFileException, ProtocolException,
MalformedURLException {
if (validate(remoteSite))
- return protocolHandler.pwd(protocolHandler
+ return protocolHandler.pwd(remoteSite, protocolHandler
.getAppropriateProtocolBySite(remoteSite, true));
else
throw new ProtocolException("Not a valid remote site " + remoteSite);
@@ -355,7 +358,7 @@ public class FileRetrievalSystem {
if (validate(remoteSite)) {
if (!file.startsWith("/"))
file = "/" + file;
- return addToDownloadQueue(protocolHandler.getProtocolFileFor(
+ return addToDownloadQueue(protocolHandler.getProtocolFileFor(remoteSite,
protocolHandler.getAppropriateProtocolBySite(remoteSite,
true), file, false), renamingString, downloadToDir,
uniqueMetadataElement, deleteAfterDownload);
@@ -364,6 +367,7 @@ public class FileRetrievalSystem {
}
public boolean validate(RemoteSite remoteSite) {
+ Preconditions.checkNotNull(remoteSite);
LinkedList<RemoteSite> remoteSites = this.siteInfo
.getPossibleRemoteSites(remoteSite.getAlias(), remoteSite
.getURL(), remoteSite.getUsername(), remoteSite
@@ -409,12 +413,12 @@ public class FileRetrievalSystem {
+ "' because it is already on the download queue");
return false;
}
-
+
RemoteFile remoteFile = new RemoteFile(file);
remoteFile.addMetadata(RemoteFile.RENAMING_STRING, renamingString);
remoteFile.addMetadata(RemoteFile.DELETE_AFTER_DOWNLOAD,
deleteAfterDownload + "");
-
+
String mimeType = this.mimeTypeDetection.getMimeType(file.getName());
if (mimeType != null
&& !mimeType.equals("application/octet-stream")) {
@@ -428,7 +432,7 @@ public class FileRetrievalSystem {
for (String field : description.split("\\&\\&")) {
String[] keyval = field.split("\\=");
remoteFile.addMetadata(keyval[0].trim(), keyval[1].trim());
- }
+ }
}
else{
// it's the ProductType
@@ -447,15 +451,15 @@ public class FileRetrievalSystem {
+ downloadToDir.getPath());
if (!this.isStagingAreaInitialized(downloadToDir))
this.initializeStagingArea(downloadToDir);
-
+
remoteFile.addMetadata(RemoteFile.DOWNLOAD_TO_DIR, downloadToDir.getAbsolutePath());
-
+
if (remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR) != null) {
remoteFile.addMetadata(RemoteFile.PRODUCT_NAME, RenamingConvention.rename(remoteFile.getProtocolFile(),
remoteFile.getMetadata(RemoteFile.PRODUCT_NAME_GENERATOR)));
}else {
remoteFile.setUniqueMetadataElement(uniqueMetadataElement == null ? RemoteFile.FILENAME
: uniqueMetadataElement);
}
-
+
if (!isAlreadyInDatabase(remoteFile)) {
// get download location
@@ -522,7 +526,8 @@ public class FileRetrievalSystem {
LOG.log(Level.INFO, "Preparing staging area " + stagingArea);
if (stagingArea.exists()) {
File[] failedDownloads = stagingArea.listFiles(new FileFilter() {
- public boolean accept(File pathname) {
+ @Override
+ public boolean accept(File pathname) {
return pathname.getName().startsWith("Downloading_");
}
});
@@ -574,7 +579,7 @@ public class FileRetrievalSystem {
/**
* Sleeps the crawling thread
- *
+ *
* @throws InterruptedException
*/
synchronized void waitMainThread() throws InterruptedException {
@@ -602,7 +607,7 @@ public class FileRetrievalSystem {
/**
* Gets an available downloading session Protocol. Returns null if none are
* available
- *
+ *
* @param path
* The session returned will be checked against the Path passed
* in and if not presently connected to the Path's URL, it will
@@ -652,7 +657,8 @@ public class FileRetrievalSystem {
if (file.isDir())
protocolHandler.cd(session, file);
else
- protocolHandler.cd(session, new RemoteSiteFile(file.getParent()));
+ protocolHandler.cd(session,
+ new RemoteSiteFile(file.getParent(), file.getSite()));
} catch (Exception e) {
e.printStackTrace();
try {
@@ -668,7 +674,7 @@ public class FileRetrievalSystem {
/**
* Puts a session in the available session list
- *
+ *
* @param session
* The Protocol session to be added to the available list
*/
@@ -678,7 +684,7 @@ public class FileRetrievalSystem {
/**
* Removes a session from the available list and returns it
- *
+ *
* @return An available downloading Protocol session
*/
synchronized Protocol getAvailableSession() {
@@ -692,7 +698,7 @@ public class FileRetrievalSystem {
/**
* Registers a downloading session with the threadpoolexecutor to begin
* downloading the specified ProtocolFile to the local File location
- *
+ *
* @param session
* The downloading Protocol session to be used to download the
* ProtocolFile
@@ -705,6 +711,7 @@ public class FileRetrievalSystem {
final RemoteFile remoteFile, final File newFile) {
this.addToDownloadingList(remoteFile.getProtocolFile());
threadController.execute(new Runnable() {
+ @Override
public void run() {
boolean successful = false;
int retries = 0;
@@ -797,7 +804,7 @@ public class FileRetrievalSystem {
LOG.log(Level.WARNING, "Retrying to download file "
+ remoteFile.getProtocolFile()
+ " because download failed : "
- + e.getMessage());
+ + e.getMessage(), e);
try {
protocolHandler.disconnect(curSession);
} catch (Exception exc) {
@@ -811,7 +818,7 @@ public class FileRetrievalSystem {
"Failed to reconnect protocol to retry download of
file "
+ remoteFile.getProtocolFile()
+ " -- aborting retry : "
- + e.getMessage());
+ + e.getMessage(), e);
}
} else {
LOG
@@ -821,7 +828,7 @@ public class FileRetrievalSystem {
+ remoteFile
.getProtocolFile()
+ " do to too many previous download
failures : "
- + e.getMessage());
+ + e.getMessage(), e);
if (FileRetrievalSystem.this.dListener != null)
FileRetrievalSystem.this.dListener
.downloadFailed(remoteFile
@@ -924,7 +931,7 @@ public class FileRetrievalSystem {
* list. The ThreadPoolExecutor needs to be completely shutdown before this
* method should be called. Otherwise some Protocols might not be
* disconnected or left downloading.
- *
+ *
* @return True if successful, false otherwise
* @throws RemoteConnectionException
*/
Modified: oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
URL: http://svn.apache.org/viewvc/oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java?rev=1366499&r1=1366498&r2=1366499&view=diff
==============================================================================
--- oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
(original)
+++ oodt/trunk/pushpull/src/main/java/org/apache/oodt/cas/pushpull/retrievalsystem/RetrievalSetup.java
Fri Jul 27 19:05:29 2012
@@ -44,27 +44,27 @@ import org.apache.oodt.cas.pushpull.retr
import org.apache.oodt.cas.pushpull.retrievalsystem.FileRetrievalSystem;
/**
- *
+ *
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* Describe your class here
* </p>.
*/
public class RetrievalSetup {
- private Config config;
+ private final Config config;
- private HashSet<File> alreadyProcessedPropFiles;
+ private final HashSet<File> alreadyProcessedPropFiles;
- private HashMap<Class<RetrievalMethod>, RetrievalMethod> classToRmMap;
+ private final HashMap<Class<RetrievalMethod>, RetrievalMethod> classToRmMap;
private boolean downloadingProps;
- private SiteInfo siteInfo;
+ private final SiteInfo siteInfo;
- private DataFileToPropFileLinker linker;
+ private final DataFileToPropFileLinker linker;
private final static Logger LOG = Logger.getLogger(RetrievalSetup.class
.getName());
@@ -110,7 +110,7 @@ public class RetrievalSetup {
linker);
} catch (ParserException e) {
LOG.log(Level.SEVERE, "Failed to parse property file "
- + propFile + " : " + e.getMessage());
+ + propFile + " : " + e.getMessage(), e);
linker.markAsFailed(propFile,
"Failed to parse property file " + propFile
+ " : " + e.getMessage());
@@ -118,7 +118,7 @@ public class RetrievalSetup {
LOG.log(Level.SEVERE,
"Failed to finish downloading per property files "
+ propFile.getAbsolutePath() + " : "
- + e.getMessage());
+ + e.getMessage(), e);
linker.markAsFailed(propFile,
"Error while downloading per property file "
+ propFile.getAbsolutePath() + " : "
@@ -160,7 +160,8 @@ public class RetrievalSetup {
this.downloadingProps = true;
new Thread(new Runnable() {
- public void run() {
+ @Override
+ public void run() {
FileRetrievalSystem frs = null;
try {
(frs = new FileRetrievalSystem(
@@ -217,6 +218,7 @@ public class RetrievalSetup {
private File[] getCurrentlyDownloadedPropFiles(final PropFilesInfo pfi) {
File[] files = pfi.getLocalDir().listFiles(new FileFilter() {
+ @Override
public boolean accept(File pathname) {
return pfi.getParserForFile(pathname) != null
&& !(pathname.getName().startsWith("Downloading_")
@@ -247,7 +249,7 @@ public class RetrievalSetup {
File newLoc = new File(moveToDir, dirstructFile.getName());
dirstructFile.renameTo(newLoc);
new File(dirstructFile.getAbsolutePath() + "." + config.getMetFileExtension())
- .renameTo(new File(newLoc.getAbsolutePath() + "."
+ .renameTo(new File(newLoc.getAbsolutePath() + "."
+ config.getMetFileExtension()));
if (errorMsgs != null) {
File errorFile = new File(newLoc.getParentFile(), dirstructFile
|