lucene-solr-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "sangraal aiken" <sangr...@gmail.com>
Subject Re: Doc add limit
Date Fri, 28 Jul 2006 00:46:32 GMT
Yeah, I'm closing them.  Here's the method:

---------
  private String doUpdate(String sw) {
    StringBuffer updateResult = new StringBuffer();
    try {
      // open connection
      log.info("Connecting to and preparing to post to SolrUpdate
servlet.");
      URL url = new URL("http://localhost:8080/update");
      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
      conn.setRequestMethod("POST");
      conn.setRequestProperty("Content-Type", "application/octet-stream");
      conn.setDoOutput(true);
      conn.setDoInput(true);
      conn.setUseCaches(false);

      // Write to server
      log.info("About to post to SolrUpdate servlet.");
      DataOutputStream output = new DataOutputStream(conn.getOutputStream
());
      output.writeBytes(sw);
      output.flush();
      output.close();
      log.info("Finished posting to SolrUpdate servlet.");

      // Read response
      log.info("Ready to read response.");
      BufferedReader rd = new BufferedReader(new InputStreamReader(
conn.getInputStream()));
      log.info("Got reader....");
      String line;
      while ((line = rd.readLine()) != null) {
        log.info("Writing to result...");
        updateResult.append(line);
      }
      rd.close();

      // close connections
      conn.disconnect();

      log.info("Done updating Solr for site" + updateSite);
    } catch (Exception e) {
      e.printStackTrace();
    }

    return updateResult.toString();
  }
}

-Sangraal

On 7/27/06, Yonik Seeley <yonik@apache.org> wrote:
>
> Are you reading the response and closing the connection?  If not, you
> are probably running out of socket connections.
>
> -Yonik
>
> On 7/27/06, sangraal aiken <sangraal@gmail.com> wrote:
> > Yonik,
> > It looks like the problem is with the way I'm posting to the SolrUpdate
> > servlet. I am able to use curl to post the data to my tomcat instance
> > without a problem. It only fails when I try to handle the http post from
> > java... my code is below:
> >
> >       URL url = new URL("http://localhost:8983/solr/update");
> >       HttpURLConnection conn = (HttpURLConnection) url.openConnection();
> >       conn.setRequestMethod("POST");
> >       conn.setRequestProperty("Content-Type",
> "application/octet-stream");
> >       conn.setDoOutput(true);
> >       conn.setDoInput(true);
> >       conn.setUseCaches(false);
> >
> >       // Write to server
> >       log.info("About to post to SolrUpdate servlet.");
> >       DataOutputStream output = new DataOutputStream(
> conn.getOutputStream
> > ());
> >       output.writeBytes(sw);
> >       output.flush();
> >       log.info("Finished posting to SolrUpdate servlet.");
> >
> > -Sangraal
> >
> > On 7/27/06, Yonik Seeley <yonik@apache.org> wrote:
> > >
> > > On 7/26/06, sangraal aiken <sangraal@gmail.com> wrote:
> > > > I removed everything from the Add xml so the docs looked like this:
> > > >
> > > > <doc>
> > > > <field name="id">187880</field>
> > > > </doc>
> > > > <doc>
> > > > <field name="id">187852</field>
> > > > </doc>
> > > >
> > > > and it still hung at 6,144...
> > >
> > > Maybe you can try the following simple Python client to try and rule
> > > out some kind of different client interactions... the attached script
> > > adds 10,000 documents and works fine for me in WinXP w/ Tomcat 5.5.17
> > > and Jetty
> > >
> > > -Yonik
> > >
> > >
> > > ------------------------------------ solr.py ----------------------
> > > import httplib
> > > import socket
> > >
> > > class SolrConnection:
> > >   def __init__(self, host='localhost:8983', solrBase='/solr'):
> > >     self.host = host
> > >     self.solrBase = solrBase
> > >     #a connection to the server is not opened at this point.
> > >     self.conn = httplib.HTTPConnection(self.host)
> > >     #self.conn.set_debuglevel(1000000)
> > >     self.postheaders = {"Connection":"close"}
> > >
> > >   def doUpdateXML(self, request):
> > >     try:
> > >       self.conn.request('POST', self.solrBase+'/update', request,
> > > self.postheaders)
> > >     except (socket.error,httplib.CannotSendRequest) :
> > >       #reconnect in case the connection was broken from the server
> going
> > > down,
> > >       #the server timing out our persistent connection, or another
> > >       #network failure.
> > >       #Also catch httplib.CannotSendRequest because the HTTPConnection
> > > object
> > >       #can get in a bad state.
> > >       self.conn.close()
> > >       self.conn.connect()
> > >       self.conn.request('POST', self.solrBase+'/update', request,
> > > self.postheaders)
> > >
> > >     rsp = self.conn.getresponse()
> > >     #print rsp.status, rsp.reason
> > >     data = rsp.read()
> > >     #print "data=",data
> > >     self.conn.close()
> > >
> > >   def delete(self, id):
> > >     xstr = '<delete><id>'+id+'</id></delete>'
> > >     self.doUpdateXML(xstr)
> > >
> > >   def add(self, **fields):
> > >     #todo: XML escaping
> > >     flist=['<field name="%s">%s</field>' % f for f in fields.items()
]
> > >     flist.insert(0,'<add><doc>')
> > >     flist.append('</doc></add>')
> > >     xstr = ''.join(flist)
> > >     self.doUpdateXML(xstr)
> > >
> > > c = SolrConnection()
> > > #for i in range(10000):
> > > #  c.delete(str(i))
> > > for i in range(10000):
> > >   c.add(id=i)
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message