lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 132 - Still Unstable!
Date Thu, 24 Aug 2017 13:22:45 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/132/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.DistributedVersionInfoTest.testReplicaVersionHandling

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50361/solr/c8n_vers_1x3_shard1_replica_n1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50361/solr/c8n_vers_1x3_shard1_replica_n1
	at __randomizedtesting.SeedInfo.seed([BBC13DCF71177D34:6738EA35D36CB775]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:641)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.DistributedVersionInfoTest.assertDocExists(DistributedVersionInfoTest.java:341)
	at org.apache.solr.cloud.DistributedVersionInfoTest.assertDocsExistInAllReplicas(DistributedVersionInfoTest.java:311)
	at org.apache.solr.cloud.DistributedVersionInfoTest.testReplicaVersionHandling(DistributedVersionInfoTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50361 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
	... 45 more


FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
No 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([BBC13DCF71177D34:E42510F81A1BEE71]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11892 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_BBC13DCF71177D34-001/init-core-data-001
   [junit4]   2> 1799106 WARN  (SUITE-SolrSlf4jReporterTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=81 numCloses=81
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=BBC13DCF71177D34 -Dtests.slow=true -Dtests.locale=de-LU -Dtests.timezone=Asia/Damascus -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 6.36s J0 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BBC13DCF71177D34:E42510F81A1BEE71]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_BBC13DCF71177D34-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=de-LU, timezone=Asia/Damascus
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=200248616,total=481566720
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsCollectionsAPIDistributedZkTest, DeleteNodeTest, TestExactStatsCache, TestUpdate, SearchHandlerTest, TestLegacyFieldReuse, TestGeoJSONResponseWriter, SoftAutoCommitTest, TestLegacyTerms, TestSubQueryTransformer, TestManagedSchemaAPI, BasicDistributedZkTest, CleanupOldIndexTest, TestZkAclsWithHadoopAuth, TestJavabinTupleStreamParser, UUIDFieldTest, TestRandomDVFaceting, SolrShardReporterTest, TestUnifiedSolrHighlighter, TestIndexingPerformance, AddBlockUpdateTest, RAMDirectoryFactoryTest, SpellCheckCollatorWithCollapseTest, TestStressReorder, RankQueryTest, SolrJmxReporterCloudTest, TestLMDirichletSimilarityFactory, TestCloudSearcherWarming, TestManagedSynonymGraphFilterFactory, DirectUpdateHandlerTest, TestDownShardTolerantSearch, TestSegmentSorting, DirectSolrConnectionTest, SolrCLIZkUtilsTest, TestLazyCores, DateMathParserTest, SolrCoreMetricManagerTest, TestBackupRepositoryFactory, TestMacros, TestPKIAuthenticationPlugin, TestSchemaSimilarityResource, TestStressLiveNodes, DefaultValueUpdateProcessorTest, SimplePostToolTest, ReturnFieldsTest, TestPhraseSuggestions, TestConfigSetImmutable, CreateCollectionCleanupTest, SharedFSAutoReplicaFailoverUtilsTest, TestHashQParserPlugin, TestFaceting, TestExpandComponent, TestHdfsCloudBackupRestore, TestTestInjection, ImplicitSnitchTest, SSLMigrationTest, BinaryUpdateRequestHandlerTest, TestCoreContainer, TestLeaderElectionZkExpiry, RecoveryZkTest, TestCloudSchemaless, TestXmlQParserPlugin, FacetPivotSmallTest, OutputWriterTest, RecoveryAfterSoftCommitTest, XmlUpdateRequestHandlerTest, HdfsUnloadDistributedZkTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSubQueryTransformerDistrib, TestNoOpRegenerator, TestReplicationHandler, PeerSyncWithIndexFingerprintCachingTest, TestFieldCache, ClassificationUpdateProcessorTest, ForceLeaderTest, ZkCLITest, TestUtils, HighlighterConfigTest, TestCustomSort, RestartWhileUpdatingTest, ScriptEngineTest, HighlighterTest, StressHdfsTest, CdcrReplicationHandlerTest, ShardSplitTest, AutoScalingHandlerTest, CircularListTest, BJQParserTest, SolrInfoBeanTest, ResponseHeaderTest, ParsingFieldUpdateProcessorsTest, HardAutoCommitTest, MoveReplicaHDFSUlogDirTest, DeleteInactiveReplicaTest, TestCloudJSONFacetJoinDomain, OverseerTest, OverseerRolesTest, ResponseLogComponentTest, TestSolrConfigHandlerConcurrent, ClusterStateTest, TestCoreAdminApis, TestTolerantSearch, TestDeleteCollectionOnDownNodes, TestDistributedSearch, TestHdfsUpdateLog, CollectionTooManyReplicasTest, TestStandardQParsers, TestHashPartitioner, SmileWriterTest, FastVectorHighlighterTest, TestConfigReload, DisMaxRequestHandlerTest, AsyncCallRequestStatusResponseTest, TestAddFieldRealTimeGet, TestPHPSerializedResponseWriter, DistributedFacetPivotWhiteBoxTest, TestCustomStream, ZkStateReaderTest, TestSort, TestInPlaceUpdatesStandalone, TestLocalFSCloudBackupRestore, TestQuerySenderNoQuery, TestCursorMarkWithoutUniqueKey, JavabinLoaderTest, TestCloudDeleteByQuery, SparseHLLTest, TestConfigSetsAPI, DistribCursorPagingTest, TestConfigSetsAPIZkFailure, TestSolrDeletionPolicy2, TestQueryWrapperFilter, AnalysisAfterCoreReloadTest, TestCloudPseudoReturnFields, TestLegacyFieldCache, TestPayloadCheckQParserPlugin, HdfsLockFactoryTest, TestConfigSets, ActionThrottleTest, TestNonDefinedSimilarityFactory, ManagedSchemaRoundRobinCloudTest, TestTrieFacet, SortByFunctionTest, ConfigureRecoveryStrategyTest, TestFunctionQuery, TestScoreJoinQPScore, MinimalSchemaTest, ChaosMonkeySafeLeaderWithPullReplicasTest, IndexSchemaTest, HdfsThreadLeakTest, WordBreakSolrSpellCheckerTest, CSVRequestHandlerTest, TlogReplayBufferedWhileIndexingTest, TestBulkSchemaConcurrent, BlockCacheTest, PrimitiveFieldTypeTest, StatsReloadRaceTest, AnalyticsQueryTest, CdcrBootstrapTest, SuggesterWFSTTest, TestDynamicFieldCollectionResource, TestManagedResourceStorage, AtomicUpdateProcessorFactoryTest, UnloadDistributedZkTest, DistributedExpandComponentTest, AssignBackwardCompatibilityTest, TestPrepRecovery, SolrIndexSplitterTest, TestFuzzyAnalyzedSuggestions, PeerSyncTest, SolrSlf4jReporterTest]
   [junit4] Completed [367/731 (1!)] on J0 in 6.39s, 1 test, 1 failure <<< FAILURES!

[...truncated 499 lines...]
   [junit4] Suite: org.apache.solr.cloud.DistributedVersionInfoTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/init-core-data-001
   [junit4]   2> 2104861 WARN  (SUITE-DistributedVersionInfoTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=14 numCloses=14
   [junit4]   2> 2104861 INFO  (SUITE-DistributedVersionInfoTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2104867 INFO  (SUITE-DistributedVersionInfoTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2104867 INFO  (SUITE-DistributedVersionInfoTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2104874 ERROR (Thread-3636) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2105053 ERROR (jetty-launcher-3954-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2105054 ERROR (jetty-launcher-3954-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2105059 ERROR (jetty-launcher-3954-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2105531 INFO  (TEST-DistributedVersionInfoTest.testReplicaVersionHandling-seed#[BBC13DCF71177D34]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReplicaVersionHandling
   [junit4]   2> 2111002 ERROR (qtp1664428395-20889) [n:127.0.0.1:50363_solr c:c8n_vers_1x3 s:shard1 r:core_node5 x:c8n_vers_1x3_shard1_replica_n2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111002 ERROR (qtp1822749906-20880) [n:127.0.0.1:50360_solr c:c8n_vers_1x3 s:shard1 r:core_node6 x:c8n_vers_1x3_shard1_replica_n4] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n4]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111002 ERROR (qtp2073496119-20877) [n:127.0.0.1:50361_solr c:c8n_vers_1x3 s:shard1 r:core_node3 x:c8n_vers_1x3_shard1_replica_n1] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n1]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111003 ERROR (qtp1822749906-20880) [n:127.0.0.1:50360_solr c:c8n_vers_1x3 s:shard1 r:core_node6 x:c8n_vers_1x3_shard1_replica_n4] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n4]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111003 ERROR (qtp1664428395-20889) [n:127.0.0.1:50363_solr c:c8n_vers_1x3 s:shard1 r:core_node5 x:c8n_vers_1x3_shard1_replica_n2] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111003 ERROR (qtp2073496119-20877) [n:127.0.0.1:50361_solr c:c8n_vers_1x3 s:shard1 r:core_node3 x:c8n_vers_1x3_shard1_replica_n1] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:113)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to reload core [c8n_vers_1x3_shard1_replica_n1]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1249)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:111)
   [junit4]   2> 	... 33 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:636)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1231)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 36 more
   [junit4]   2> 
   [junit4]   2> 2111004 ERROR (OverseerThreadFactory-8531-thread-2-processing-n:127.0.0.1:50360_solr) [n:127.0.0.1:50360_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:50360/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50360/solr: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2111004 ERROR (OverseerThreadFactory-8531-thread-2-processing-n:127.0.0.1:50360_solr) [n:127.0.0.1:50360_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:50363/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50363/solr: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2111004 ERROR (OverseerThreadFactory-8531-thread-2-processing-n:127.0.0.1:50360_solr) [n:127.0.0.1:50360_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:50361/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50361/solr: Error handling 'reload' action
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2143215 ERROR (qtp1822749906-20874) [n:127.0.0.1:50360_solr c:c8n_vers_1x3 s:shard1 r:core_node6 x:c8n_vers_1x3_shard1_replica_n4] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files)
   [junit4]   2> 	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:193)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:446)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1258)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:532)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:517)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:347)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:266)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:216)
   [junit4]   2> 	at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
   [junit4]   2> 	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:991)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1207)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:753)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
   [junit4]   2> 	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
   [junit4]   2> 	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files)
   [junit4]   2> 	at java.io.RandomAccessFile.open0(Native Method)
   [junit4]   2> 	at java.io.RandomAccessFile.open(RandomAccessFile.java:316)
   [junit4]   2> 	at java.io.RandomAccessFile.<init>(RandomAccessFile.java:243)
   [junit4]   2> 	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:159)
   [junit4]   2> 	... 56 more
   [junit4]   2> 
   [junit4]   2> 2143216 ERROR (qtp1822749906-20874) [n:127.0.0.1:50360_solr c:c8n_vers_1x3 s:shard1 r:core_node6 x:c8n_vers_1x3_shard1_replica_n4] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files)
   [junit4]   2> 	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:193)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:446)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1258)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:532)
   [junit4]   2> 	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:517)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:347)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:266)
   [junit4]   2> 	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:216)
   [junit4]   2> 	at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:67)
   [junit4]   2> 	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:991)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:1207)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:753)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
   [junit4]   2> 	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
   [junit4]   2> 	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files)
   [junit4]   2> 	at java.io.RandomAccessFile.open0(Native Method)
   [junit4]   2> 	at java.io.RandomAccessFile.open(RandomAccessFile.java:316)
   [junit4]   2> 	at java.io.RandomAccessFile.<init>(RandomAccessFile.java:243)
   [junit4]   2> 	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:159)
   [junit4]   2> 	... 56 more
   [junit4]   2> 
   [junit4]   2> 2143216 ERROR (TEST-DistributedVersionInfoTest.testReplicaVersionHandling-seed#[BBC13DCF71177D34]) [    ] o.a.s.c.s.i.CloudSolrClient Request to collection c8n_vers_1x3 failed due to (500) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50360/solr/c8n_vers_1x3_shard1_replica_n4: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files), retry? 0
   [junit4]   2> 2143216 WARN  (TEST-DistributedVersionInfoTest.testReplicaVersionHandling-seed#[BBC13DCF71177D34]) [    ] o.a.s.c.AbstractFullDistribZkTestBase ERROR: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50360/solr/c8n_vers_1x3_shard1_replica_n4: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001/tempDir-001/node3/c8n_vers_1x3_shard1_replica_n4/data/tlog/tlog.0000000000000000000 (Too many open files) ... Sleeping for 100 seconds before re-try ...
   [junit4]   2> 2243280 WARN  (qtp2073496119-20875-acceptor-0@19f8291d-ServerConnector@173fc452{HTTP/1.1,[http/1.1]}{127.0.0.1:50361}) [    ] o.e.j.s.AbstractConnector 
   [junit4]   2> java.io.IOException: Too many open files
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
   [junit4]   2> 	at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2246284 INFO  (TEST-DistributedVersionInfoTest.testReplicaVersionHandling-seed#[BBC13DCF71177D34]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReplicaVersionHandling
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedVersionInfoTest -Dtests.method=testReplicaVersionHandling -Dtests.seed=BBC13DCF71177D34 -Dtests.slow=true -Dtests.locale=es-SV -Dtests.timezone=Asia/Omsk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    141s J0 | DistributedVersionInfoTest.testReplicaVersionHandling <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50361/solr/c8n_vers_1x3_shard1_replica_n1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BBC13DCF71177D34:6738EA35D36CB775]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:641)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    > 	at org.apache.solr.cloud.DistributedVersionInfoTest.assertDocExists(DistributedVersionInfoTest.java:341)
   [junit4]    > 	at org.apache.solr.cloud.DistributedVersionInfoTest.assertDocsExistInAllReplicas(DistributedVersionInfoTest.java:311)
   [junit4]    > 	at org.apache.solr.cloud.DistributedVersionInfoTest.testReplicaVersionHandling(DistributedVersionInfoTest.java:102)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50361 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
   [junit4]    > 	... 45 more
   [junit4]   2> 2247949 WARN  (zkCallback-3971-thread-4-processing-n:127.0.0.1:50361_solr) [n:127.0.0.1:50361_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2247951 WARN  (zkCallback-3969-thread-5-processing-n:127.0.0.1:50363_solr) [n:127.0.0.1:50363_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2247963 WARN  (zkCallback-3970-thread-3-processing-n:127.0.0.1:50360_solr) [n:127.0.0.1:50360_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2247964 ERROR (SUITE-DistributedVersionInfoTest-seed#[BBC13DCF71177D34]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2248827 WARN  (Thread-3636) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	3	/solr/configs/conf
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	22	/solr/collections/c8n_vers_1x3/state.json
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/collections/c8n_vers_1x3/leader_elect/shard1/election/98538565424775176-core_node6-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/98538565424775176-127.0.0.1:50360_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.DistributedVersionInfoTest_BBC13DCF71177D34-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {_version_=PostingsFormat(name=LuceneVarGapFixedInterval), a_t=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Memory)}, docValues:{}, maxPointsInLeafNode=1712, maxMBSortInHeap=7.419883556101515, sim=RandomSimilarity(queryNorm=false): {}, locale=es-SV, timezone=Asia/Omsk
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=160059048,total=461168640
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsCollectionsAPIDistributedZkTest, DeleteNodeTest, TestExactStatsCache, TestUpdate, SearchHandlerTest, TestLegacyFieldReuse, TestGeoJSONResponseWriter, SoftAutoCommitTest, TestLegacyTerms, TestSubQueryTransformer, TestManagedSchemaAPI, BasicDistributedZkTest, CleanupOldIndexTest, TestZkAclsWithHadoopAuth, TestJavabinTupleStreamParser, UUIDFieldTest, TestRandomDVFaceting, SolrShardReporterTest, TestUnifiedSolrHighlighter, TestIndexingPerformance, AddBlockUpdateTest, RAMDirectoryFactoryTest, SpellCheckCollatorWithCollapseTest, TestStressReorder, RankQueryTest, SolrJmxReporterCloudTest, TestLMDirichletSimilarityFactory, TestCloudSearcherWarming, TestManagedSynonymGraphFilterFactory, DirectUpdateHandlerTest, TestDownShardTolerantSearch, TestSegmentSorting, DirectSolrConnectionTest, SolrCLIZkUtilsTest, TestLazyCores, DateMathParserTest, SolrCoreMetricManagerTest, TestBackupRepositoryFactory, TestMacros, TestPKIAuthenticationPlugin, TestSchemaSimilarityResource, TestStressLiveNodes, DefaultValueUpdateProcessorTest, SimplePostToolTest, ReturnFieldsTest, TestPhraseSuggestions, TestConfigSetImmutable, CreateCollectionCleanupTest, SharedFSAutoReplicaFailoverUtilsTest, TestHashQParserPlugin, TestFaceting, TestExpandComponent, TestHdfsCloudBackupRestore, TestTestInjection, ImplicitSnitchTest, SSLMigrationTest, BinaryUpdateRequestHandlerTest, TestCoreContainer, TestLeaderElectionZkExpiry, RecoveryZkTest, TestCloudSchemaless, TestXmlQParserPlugin, FacetPivotSmallTest, OutputWriterTest, RecoveryAfterSoftCommitTest, XmlUpdateRequestHandlerTest, HdfsUnloadDistributedZkTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSubQueryTransformerDistrib, TestNoOpRegenerator, TestReplicationHandler, PeerSyncWithIndexFingerprintCachingTest, TestFieldCache, ClassificationUpdateProcessorTest, ForceLeaderTest, ZkCLITest, TestUtils, HighlighterConfigTest, TestCustomSort, RestartWhileUpdatingTest, ScriptEngineTest, HighlighterTest, StressHdfsTest, CdcrReplicationHandlerTest, ShardSplitTest, AutoScalingHandlerTest, CircularListTest, BJQParserTest, SolrInfoBeanTest, ResponseHeaderTest, ParsingFieldUpdateProcessorsTest, HardAutoCommitTest, MoveReplicaHDFSUlogDirTest, DeleteInactiveReplicaTest, TestCloudJSONFacetJoinDomain, OverseerTest, OverseerRolesTest, ResponseLogComponentTest, TestSolrConfigHandlerConcurrent, ClusterStateTest, TestCoreAdminApis, TestTolerantSearch, TestDeleteCollectionOnDownNodes, TestDistributedSearch, TestHdfsUpdateLog, CollectionTooManyReplicasTest, TestStandardQParsers, TestHashPartitioner, SmileWriterTest, FastVectorHighlighterTest, TestConfigReload, DisMaxRequestHandlerTest, AsyncCallRequestStatusResponseTest, TestAddFieldRealTimeGet, TestPHPSerializedResponseWriter, DistributedFacetPivotWhiteBoxTest, TestCustomStream, ZkStateReaderTest, TestSort, TestInPlaceUpdatesStandalone, TestLocalFSCloudBackupRestore, TestQuerySenderNoQuery, TestCursorMarkWithoutUniqueKey, JavabinLoaderTest, TestCloudDeleteByQuery, SparseHLLTest, TestConfigSetsAPI, DistribCursorPagingTest, TestConfigSetsAPIZkFailure, TestSolrDeletionPolicy2, TestQueryWrapperFilter, AnalysisAfterCoreReloadTest, TestCloudPseudoReturnFields, TestLegacyFieldCache, TestPayloadCheckQParserPlugin, HdfsLockFactoryTest, TestConfigSets, ActionThrottleTest, TestNonDefinedSimilarityFactory, ManagedSchemaRoundRobinCloudTest, TestTrieFacet, SortByFunctionTest, ConfigureRecoveryStrategyTest, TestFunctionQuery, TestScoreJoinQPScore, MinimalSchemaTest, ChaosMonkeySafeLeaderWithPullReplicasTest, IndexSchemaTest, HdfsThreadLeakTest, WordBreakSolrSpellCheckerTest, CSVRequestHandlerTest, TlogReplayBufferedWhileIndexingTest, TestBulkSchemaConcurrent, BlockCacheTest, PrimitiveFieldTypeTest, StatsReloadRaceTest, AnalyticsQueryTest, CdcrBootstrapTest, SuggesterWFSTTest, TestDynamicFieldCollectionResource, TestManagedResourceStorage, AtomicUpdateProcessorFactoryTest, UnloadDistributedZkTest, DistributedExpandComponentTest, AssignBackwardCompatibilityTest, TestPrepRecovery, SolrIndexSplitterTest, TestFuzzyAnalyzedSuggestions, PeerSyncTest, SolrSlf4jReporterTest, CursorPagingTest, TestXIncludeConfig, TestSolrQueryResponse, TestRandomRequestDistribution, TestClassicSimilarityFactory, CdcrVersionReplicationTest, SolrXmlInZkTest, HdfsRestartWhileUpdatingTest, TestRangeQuery, TestSolr4Spatial, TestGroupingSearch, QueryElevationComponentTest, BadIndexSchemaTest, TestBadConfig, TestIndexSearcher, TestCoreDiscovery, SpellCheckCollatorTest, TestFoldingMultitermQuery, DocValuesTest, SpatialFilterTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, TestSurroundQueryParser, TestWriterPerf, TestQueryTypes, FileBasedSpellCheckerTest, DocumentBuilderTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, SolrPluginUtilsTest, TestCollationField, MBeansHandlerTest, SOLR749Test, TestQuerySenderListener, TestStressRecovery, SpellPossibilityIteratorTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestFastWriter, TimeZoneUtilsTest, TestFastOutputStream, PluginInfoTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, CollectionReloadTest, CollectionsAPISolrJTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, DeleteLastCustomShardedReplicaTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest]
   [junit4] Completed [519/731 (2!)] on J0 in 143.99s, 1 test, 1 error <<< FAILURES!

[...truncated 44119 lines...]

Mime
View raw message