lucene-solr-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "sangraal aiken" <sangr...@gmail.com>
Subject Re: Doc add limit
Date Fri, 28 Jul 2006 14:01:39 GMT
Sure, the method that does all the work updating Solr is the doUpdate(String
s) method in the GanjaUpdate class I'm pasting below. It's hanging when I
try to read the response... the last output I receive in my log is Got
Reader...

----------

package com.iceninetech.solr.update;

import com.iceninetech.xml.XMLTransformer;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.logging.Logger;

public class GanjaUpdate {

  private String updateSite = "";
  private String XSL_URL = "http://localhost:8080/xsl/ganja.xsl";

  private static final File xmlStorageDir = new
File("/source/solr/xml-dls/");

  final Logger log = Logger.getLogger(GanjaUpdate.class.getName());

  public GanjaUpdate(String siteName) {
    this.updateSite = siteName;
    log.info("GanjaUpdate is primed and ready to update " + siteName);
  }

  public void update() {
    StringWriter sw = new StringWriter();

    try {
      // transform gawkerInput XML to SOLR update XML
      XMLTransformer transform = new XMLTransformer();
      log.info("About to transform ganjaInput XML to Solr Update XML");
      transform.transform(getXML(), sw, getXSL());
      log.info("Completed ganjaInput/SolrUpdate XML transform");

      // Write transformed XML to Disk.
      File transformedXML = new File(xmlStorageDir, updateSite+".sml");
      FileWriter fw = new FileWriter(transformedXML);
      fw.write(sw.toString());
      fw.close();

      // post to Solr
      log.info("About to update Solr for site " + updateSite);
      String result = this.doUpdate(sw.toString());
      log.info("Solr says: " + result);
      sw.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  public File getXML() {
    String XML_URL = "http://localhost:8080/" + updateSite + "/ganja-
full.xml";

    // check for file
    File localXML = new File(xmlStorageDir, updateSite + ".xml");

    try {
      if (localXML.createNewFile() && localXML.canWrite()) {
        // open connection
        log.info("Downloading: " + XML_URL);
        URL url = new URL(XML_URL);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");

        // Read response to File
        log.info("Storing XML to File" + localXML.getCanonicalPath());
        FileOutputStream fos = new FileOutputStream(new File(xmlStorageDir,
updateSite + ".xml"));

        BufferedReader rd = new BufferedReader(new InputStreamReader(
conn.getInputStream()));
        String line;
        while ((line = rd.readLine()) != null) {
          line = line + '\n'; // add break after each line. It preserves
formatting.
          fos.write(line.getBytes("UTF8"));
        }

        // close connections
        rd.close();
        fos.close();
        conn.disconnect();
        log.info("Got the XML... File saved.");
      }
    } catch (Exception e) {
      e.printStackTrace();
    }

    return localXML;
  }

  public File getXSL() {
    StringBuffer retVal = new StringBuffer();

    // check for file
    File localXSL = new File(xmlStorageDir, "ganja.xsl");

    try {
      if (localXSL.createNewFile() && localXSL.canWrite()) {
        // open connection
        log.info("Downloading: " + XSL_URL);
        URL url = new URL(XSL_URL);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        // Read response
        BufferedReader rd = new BufferedReader(new InputStreamReader(
conn.getInputStream()));
        String line;
        while ((line = rd.readLine()) != null) {
          line = line + '\n';
          retVal.append(line);
        }
        // close connections
        rd.close();
        conn.disconnect();

        log.info("Got the XSLT.");

        // output file
        log.info("Storing XSL to File" + localXSL.getCanonicalPath());
        FileOutputStream fos = new FileOutputStream(new File(xmlStorageDir,
"ganja.xsl"));
        fos.write(retVal.toString().getBytes());
        fos.close();
        log.info("File saved.");
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
    return localXSL;
  }

  private String doUpdate(String sw) {
    StringBuffer updateResult = new StringBuffer();
    try {
      // open connection
      log.info("Connecting to and preparing to post to SolrUpdate
servlet.");
      URL url = new URL("http://localhost:8080/update");
      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
      conn.setRequestMethod("POST");
      conn.setRequestProperty("Content-Type", "application/octet-stream");
      conn.setDoOutput(true);
      conn.setDoInput(true);
      conn.setUseCaches(false);

      // Write to server
      log.info("About to post to SolrUpdate servlet.");
      DataOutputStream output = new DataOutputStream(conn.getOutputStream
());
      output.writeBytes(sw);
      output.flush();
      output.close();
      log.info("Finished posting to SolrUpdate servlet.");

      // Read response
      log.info("Ready to read response.");
      BufferedReader rd = new BufferedReader(new InputStreamReader(
conn.getInputStream()));
      log.info("Got reader....");
      String line;
      while ((line = rd.readLine()) != null) {
        log.info("Writing to result...");
        updateResult.append(line);
      }
      rd.close();

      // close connections
      conn.disconnect();

      log.info("Done updating Solr for site" + updateSite);
    } catch (Exception e) {
      e.printStackTrace();
    }

    return updateResult.toString();
  }
}


On 7/28/06, Chris Hostetter <hossman_lucene@fucit.org> wrote:
>
>
> : I'm sure... it seems like solr is having trouble writing to a tomcat
> : response that's been inactive for a bit. It's only 30 seconds though, so
> I'm
> : not entirely sure why that would happen.
>
> but didn't you say you don't have this problem when you use curl -- just
> your java client code?
>
> Did you try Yonik's python test client? or the java client in Jira?
>
> looking over the java clinet codey you sent, it's not clear if you are
> reading the response back, or closing the connections ... can you post a
> more complete sample app thatexhibits the problem for you?
>
>
>
> -Hoss
>
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message