lucene-solr-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Carl Roberts <carl.roberts.zap...@gmail.com>
Subject Re: Cannot reindex to add a new field
Date Tue, 27 Jan 2015 22:47:08 GMT
You are right - I just checked and now the other field 
(vulnerable-software) that is using the same xpath has blank values.  
BTW - It looks like this also works:

<field column="product" sourceColName="vulnerable-software" 
commonField="false" regex=":" replaceWith=" "/>

Here are the results for one row in json:


   "responseHeader":{
     "status":0,
     "QTime":1,
     "params":{
       "fl":"*",
       "indent":"true",
       "start":"0",
       "q":"*:*",
       "wt":"json",
       "rows":"1"}},
   "response":{"numFound":6717,"start":0,"docs":[
       {
         "product":["cpe /o freebsd freebsd 2.2.8",
           "cpe /o freebsd freebsd 1.1.5.1",
           "cpe /o freebsd freebsd 2.2.3",
           "cpe /o freebsd freebsd 2.2.2",
           "cpe /o freebsd freebsd 2.2.5",
           "cpe /o freebsd freebsd 2.2.4",
           "cpe /o freebsd freebsd 2.0.5",
           "cpe /o freebsd freebsd 2.2.6",
           "cpe /o freebsd freebsd 2.1.6.1",
           "cpe /o freebsd freebsd 2.0.1",
           "cpe /o freebsd freebsd 2.2",
           "cpe /o freebsd freebsd 2.0",
           "cpe /o openbsd openbsd 2.3",
           "cpe /o freebsd freebsd 3.0",
           "cpe /o freebsd freebsd 1.1",
           "cpe /o freebsd freebsd 2.1.6",
           "cpe /o openbsd openbsd 2.4",
           "cpe /o bsdi bsd_os 3.1",
           "cpe /o freebsd freebsd 1.0",
           "cpe /o freebsd freebsd 2.1.7",
           "cpe /o freebsd freebsd 1.2",
           "cpe /o freebsd freebsd 2.1.5",
           "cpe /o freebsd freebsd 2.1.7.1",
           "cpe /o freebsd freebsd 2.2.8",
           "cpe /o freebsd freebsd 1.1.5.1",
           "cpe /o freebsd freebsd 2.2.3",
           "cpe /o freebsd freebsd 2.2.2",
           "cpe /o freebsd freebsd 2.2.5",
           "cpe /o freebsd freebsd 2.2.4",
           "cpe /o freebsd freebsd 2.0.5",
           "cpe /o freebsd freebsd 2.2.6",
           "cpe /o freebsd freebsd 2.1.6.1",
           "cpe /o freebsd freebsd 2.0.1",
           "cpe /o freebsd freebsd 2.2",
           "cpe /o freebsd freebsd 2.0",
           "cpe /o openbsd openbsd 2.3",
           "cpe /o freebsd freebsd 3.0",
           "cpe /o freebsd freebsd 1.1",
           "cpe /o freebsd freebsd 2.1.6",
           "cpe /o openbsd openbsd 2.4",
           "cpe /o bsdi bsd_os 3.1",
           "cpe /o freebsd freebsd 1.0",
           "cpe /o freebsd freebsd 2.1.7",
           "cpe /o freebsd freebsd 1.2",
           "cpe /o freebsd freebsd 2.1.5",
           "cpe /o freebsd freebsd 2.1.7.1"],
         "id":"CVE-1999-0001",
         "summary":"ip_input.c in BSD-derived TCP/IP implementations 
allows remote attackers to cause a denial of service (crash or hang) via 
crafted packets.",
         "vulnerable-configuration":["cpe:/o:bsdi:bsd_os:3.1",
           "cpe:/o:freebsd:freebsd:1.0",
           "cpe:/o:freebsd:freebsd:1.1",
           "cpe:/o:freebsd:freebsd:1.1.5.1",
           "cpe:/o:freebsd:freebsd:1.2",
           "cpe:/o:freebsd:freebsd:2.0",
           "cpe:/o:freebsd:freebsd:2.0.5",
           "cpe:/o:freebsd:freebsd:2.1.5",
           "cpe:/o:freebsd:freebsd:2.1.6",
           "cpe:/o:freebsd:freebsd:2.1.6.1",
           "cpe:/o:freebsd:freebsd:2.1.7",
           "cpe:/o:freebsd:freebsd:2.1.7.1",
           "cpe:/o:freebsd:freebsd:2.2",
           "cpe:/o:freebsd:freebsd:2.2.3",
           "cpe:/o:freebsd:freebsd:2.2.4",
           "cpe:/o:freebsd:freebsd:2.2.5",
           "cpe:/o:freebsd:freebsd:2.2.6",
           "cpe:/o:freebsd:freebsd:2.2.8",
           "cpe:/o:freebsd:freebsd:3.0",
           "cpe:/o:openbsd:openbsd:2.3",
           "cpe:/o:openbsd:openbsd:2.4",
           "cpe:/o:freebsd:freebsd:2.2.2",
           "cpe:/o:freebsd:freebsd:2.0.1"],
         "cve":"CVE-1999-0001",
         "cwe":"CWE-20",
         "published":"1999-12-30T00:00:00.000-05:00",
         "vulnerable-software":["cpe:/o:freebsd:freebsd:2.2.8",
           "cpe:/o:freebsd:freebsd:1.1.5.1",
           "cpe:/o:freebsd:freebsd:2.2.3",
           "cpe:/o:freebsd:freebsd:2.2.2",
           "cpe:/o:freebsd:freebsd:2.2.5",
           "cpe:/o:freebsd:freebsd:2.2.4",
           "cpe:/o:freebsd:freebsd:2.0.5",
           "cpe:/o:freebsd:freebsd:2.2.6",
           "cpe:/o:freebsd:freebsd:2.1.6.1",
           "cpe:/o:freebsd:freebsd:2.0.1",
           "cpe:/o:freebsd:freebsd:2.2",
           "cpe:/o:freebsd:freebsd:2.0",
           "cpe:/o:openbsd:openbsd:2.3",
           "cpe:/o:freebsd:freebsd:3.0",
           "cpe:/o:freebsd:freebsd:1.1",
           "cpe:/o:freebsd:freebsd:2.1.6",
           "cpe:/o:openbsd:openbsd:2.4",
           "cpe:/o:bsdi:bsd_os:3.1",
           "cpe:/o:freebsd:freebsd:1.0",
           "cpe:/o:freebsd:freebsd:2.1.7",
           "cpe:/o:freebsd:freebsd:1.2",
           "cpe:/o:freebsd:freebsd:2.1.5",
           "cpe:/o:freebsd:freebsd:2.1.7.1"],
         "modified":"2010-12-16T00:00:00.000-05:00",
         "_version_":1491493094540967936}]
   }}

On 1/27/15, 5:19 PM, Alexandre Rafalovitch wrote:
> One xpath per field definition. You had two fields for the same xpath.
> If they were the same value, the best bet would be to deal with it via
> copyField in the schema.
>
> No idea why regex thing makes a difference, are you sure the other
> field is also still being indexed?
>
> Regards,
>     Alex.
>
> ----
> Sign up for my Solr resources newsletter at http://www.solr-start.com/
>
>
> On 27 January 2015 at 17:11, Carl Roberts <carl.roberts.zapata@gmail.com> wrote:
>> Well - I got this to work.  Noticed that when log4j is enabled product-info
>> was in the import as product-info=[], so I then played with the field and
>> got this definition to work in the rss-data-config.xml file:
>>
>> <field column="product-info"
>> xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"
>> regex=":" replaceWith=" "/>
>>
>> Don't ask me why the other one didn't work, as I think it should have worked
>> also.
>>
>>
>> On 1/27/15, 3:42 PM, Carl Roberts wrote:
>>> Hi,
>>>
>>> I have tried to reindex to add a new field named product-info and no
>>> matter what I do, I cannot get the new field to appear in the index after
>>> import via DIH.
>>>
>>> Here is the rss-data-config.xml configuration (field product-info is the
>>> new field I added):
>>>
>>> <dataConfig>
>>>      <dataSource type="ZIPURLDataSource" connectionTimeout="15000"
>>> readTimeout="30000"/>
>>>      <document>
>>>          <entity name="cve-2002"
>>>                  pk="id"
>>>
>>> url="https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml.zip"
>>>                  processor="XPathEntityProcessor"
>>>                  forEach="/nvd/entry"
>>>                  transformer="RegexTransformer">
>>>              <field column="id" xpath="/nvd/entry/@id" commonField="false"
>>> />
>>>              <field column="cve" xpath="/nvd/entry/cve-id"
>>> commonField="false" />
>>>              <field column="cwe" xpath="/nvd/entry/cwe/@id"
>>> commonField="false" />
>>> *<field column="product-info"
>>> xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"/>*
>>>              <field column="vulnerable-configuration"
>>> xpath="/nvd/entry/vulnerable-configuration/logical-test/fact-ref/@name"
>>> commonField="false"/>
>>>              <field column="vulnerable-software"
>>> xpath="/nvd/entry/vulnerable-software-list/product" commonField="false"/>
>>>              <field column="published"
>>> xpath="/nvd/entry/published-datetime" commonField="false" />
>>>              <field column="modified"
>>> xpath="/nvd/entry/last-modified-datetime" commonField="false" />
>>>              <field column="summary" xpath="/nvd/entry/summary"
>>> commonField="false" />
>>>          </entity>
>>>      </document>
>>> </dataConfig>
>>>
>>>
>>> Here is the section that contains the new product-info field in
>>> schema.xml:
>>>
>>>   <field name="text" type="text_general" indexed="true" stored="false"
>>> multiValued="true"/>
>>>     <field name="_version_" type="long" indexed="true" stored="true"/>
>>>     <field name="id" type="text_general" indexed="true" stored="true"/>
>>>     <field name="cve" type="text_general" indexed="true" stored="true"/>
>>>     <field name="cwe" type="text_general" indexed="true" stored="true"/>
>>> *<field name="product-info" type="text_general" indexed="true"
>>> stored="true" multiValued="true"/>*
>>>     <field name="vulnerable-configuration" type="text_general"
>>> indexed="true" stored="true" multiValued="true"/>
>>>     <field name="vulnerable-software" type="text_general" indexed="true"
>>> stored="true" multiValued="true"/>
>>>     <field name="published" type="text_general" indexed="true"
>>> stored="true" />
>>>     <field name="modified" type="text_general" indexed="true" stored="true"
>>> />
>>>     <field name="summary" type="text_general" indexed="true" stored="true"
>>> />
>>>
>>> Field product-info is defined in the same manner as vulnerable-software
>>> field and it pulls the same data as vulnerable-software list field via the
>>> same xpath, yet vulnerable-software field shows up in the results and
>>> product-info field does not.  Here is the response for a query after the
>>> import takes place - field product-info is missing:
>>>
>>> ~/dev/solr-4.10.3$ curl
>>> "http://localhost:8983/solr/nvd-rss/select?wt=json&indent=true&q=*:*&start=0&&rows=1"
>>> {
>>>    "responseHeader":{
>>>      "status":0,
>>>      "QTime":1,
>>>      "params":{
>>>        "indent":"true",
>>>        "start":"0",
>>>        "q":"*:*",
>>>        "wt":"json",
>>>        "rows":"1"}},
>>>    "response":{"numFound":6717,"start":0,"docs":[
>>>        {
>>>          "id":"CVE-1999-0001",
>>>          "summary":"ip_input.c in BSD-derived TCP/IP implementations allows
>>> remote attackers to cause a denial of service (crash or hang) via crafted
>>> packets.",
>>>          "vulnerable-configuration":["cpe:/o:bsdi:bsd_os:3.1",
>>>            "cpe:/o:freebsd:freebsd:1.0",
>>>            "cpe:/o:freebsd:freebsd:1.1",
>>>            "cpe:/o:freebsd:freebsd:1.1.5.1",
>>>            "cpe:/o:freebsd:freebsd:1.2",
>>>            "cpe:/o:freebsd:freebsd:2.0",
>>>            "cpe:/o:freebsd:freebsd:2.0.5",
>>>            "cpe:/o:freebsd:freebsd:2.1.5",
>>>            "cpe:/o:freebsd:freebsd:2.1.6",
>>>            "cpe:/o:freebsd:freebsd:2.1.6.1",
>>>            "cpe:/o:freebsd:freebsd:2.1.7",
>>>            "cpe:/o:freebsd:freebsd:2.1.7.1",
>>>            "cpe:/o:freebsd:freebsd:2.2",
>>>            "cpe:/o:freebsd:freebsd:2.2.3",
>>>            "cpe:/o:freebsd:freebsd:2.2.4",
>>>            "cpe:/o:freebsd:freebsd:2.2.5",
>>>            "cpe:/o:freebsd:freebsd:2.2.6",
>>>            "cpe:/o:freebsd:freebsd:2.2.8",
>>>            "cpe:/o:freebsd:freebsd:3.0",
>>>            "cpe:/o:openbsd:openbsd:2.3",
>>>            "cpe:/o:openbsd:openbsd:2.4",
>>>            "cpe:/o:freebsd:freebsd:2.2.2",
>>>            "cpe:/o:freebsd:freebsd:2.0.1"],
>>>          "cve":"CVE-1999-0001",
>>>          "cwe":"CWE-20",
>>>          "published":"1999-12-30T00:00:00.000-05:00",
>>>          "vulnerable-software":["cpe:/o:freebsd:freebsd:2.2.8",
>>>            "cpe:/o:freebsd:freebsd:1.1.5.1",
>>>            "cpe:/o:freebsd:freebsd:2.2.3",
>>>            "cpe:/o:freebsd:freebsd:2.2.2",
>>>            "cpe:/o:freebsd:freebsd:2.2.5",
>>>            "cpe:/o:freebsd:freebsd:2.2.4",
>>>            "cpe:/o:freebsd:freebsd:2.0.5",
>>>            "cpe:/o:freebsd:freebsd:2.2.6",
>>>            "cpe:/o:freebsd:freebsd:2.1.6.1",
>>>            "cpe:/o:freebsd:freebsd:2.0.1",
>>>            "cpe:/o:freebsd:freebsd:2.2",
>>>            "cpe:/o:freebsd:freebsd:2.0",
>>>            "cpe:/o:openbsd:openbsd:2.3",
>>>            "cpe:/o:freebsd:freebsd:3.0",
>>>            "cpe:/o:freebsd:freebsd:1.1",
>>>            "cpe:/o:freebsd:freebsd:2.1.6",
>>>            "cpe:/o:openbsd:openbsd:2.4",
>>>            "cpe:/o:bsdi:bsd_os:3.1",
>>>            "cpe:/o:freebsd:freebsd:1.0",
>>>            "cpe:/o:freebsd:freebsd:2.1.7",
>>>            "cpe:/o:freebsd:freebsd:1.2",
>>>            "cpe:/o:freebsd:freebsd:2.1.5",
>>>            "cpe:/o:freebsd:freebsd:2.1.7.1"],
>>>          "modified":"2010-12-16T00:00:00.000-05:00",
>>>          "_version_":1491484873657942016}]
>>>    }}
>>>
>>> This is what I have tried so far:
>>>
>>> Restarted Solr to reload the schema and reimported via full-import and
>>> clean=true.
>>> Invoked the following command to reload the schema and reimported via
>>> full-import and clean=true
>>>
>>>      curl
>>> "http://localhost:8983/solr/admin/cores?action=RELOAD&core=nvd-rss"
>>>
>>> I am not sure why this is not working as everything seems correct to me in
>>> my setup.  Could this be a bug?
>>>
>>> Regards,
>>>
>>> Joe
>>>
>>>


Mime
View raw message