Github user jolynch commented on a diff in the pull request:
https://github.com/apache/cassandra/pull/283#discussion_r236752030
--- Diff: test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java ---
@@ -60,57 +130,229 @@ private static EndpointsForRange full(InetAddressAndPort... endpoints)
}
@Test
- public void testSnitch() throws InterruptedException, IOException, ConfigurationException
+ public void testSortedByProximity() throws InterruptedException, IOException, ConfigurationException
{
- // do this because SS needs to be initialized before DES can work properly.
- StorageService.instance.unsafeInitialize();
- SimpleSnitch ss = new SimpleSnitch();
- DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode()));
- InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
- InetAddressAndPort host1 = InetAddressAndPort.getByName("127.0.0.2");
- InetAddressAndPort host2 = InetAddressAndPort.getByName("127.0.0.3");
- InetAddressAndPort host3 = InetAddressAndPort.getByName("127.0.0.4");
- InetAddressAndPort host4 = InetAddressAndPort.getByName("127.0.0.5");
- List<InetAddressAndPort> hosts = Arrays.asList(host1, host2, host3);
+ InetAddressAndPort self = hosts[0];
+ List<InetAddressAndPort> allHosts = Arrays.asList(hosts[1], hosts[2], hosts[3]);
// first, make all hosts equal
- setScores(dsnitch, 1, hosts, 10, 10, 10);
- EndpointsForRange order = full(host1, host2, host3);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
-
- // make host1 a little worse
- setScores(dsnitch, 1, hosts, 20, 10, 10);
- order = full(host2, host3, host1);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
-
- // make host2 as bad as host1
- setScores(dsnitch, 2, hosts, 15, 20, 10);
- order = full(host3, host1, host2);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
-
- // make host3 the worst
- setScores(dsnitch, 3, hosts, 10, 10, 30);
- order = full(host1, host2, host3);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
-
- // make host3 equal to the others
- setScores(dsnitch, 5, hosts, 10, 10, 10);
- order = full(host1, host2, host3);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
+ setScores(dsnitch, 1, allHosts, 10, 10, 10);
+ EndpointsForRange order = full(hosts[1], hosts[2], hosts[3]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
+
+ // make hosts[1] a little worse
+ setScores(dsnitch, 2, allHosts, 20, 10, 10);
+ order = full(hosts[2], hosts[3], hosts[1]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
+
+ // make hosts[2] as bad as hosts[1]
+ setScores(dsnitch, 4, allHosts, 15, 20, 10);
+ order = full(hosts[3], hosts[1], hosts[2]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
+
+ // make hosts[3] the worst
+ setScores(dsnitch, 10, allHosts, 10, 10, 30);
+ order = full(hosts[1], hosts[2], hosts[3]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
+
+ // make hosts[3] equal to the others
+ setScores(dsnitch, 15, allHosts, 10, 10, 10);
+ order = full(hosts[1], hosts[2], hosts[3]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
/// Tests CASSANDRA-6683 improvements
// make the scores differ enough from the ideal order that we sort by score;
under the old
// dynamic snitch behavior (where we only compared neighbors), these wouldn't
get sorted
- setScores(dsnitch, 20, hosts, 10, 70, 20);
- order = full(host1, host3, host2);
- Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(host1, host2,
host3)));
+ dsnitch.reset();
+ setScores(dsnitch, 20, allHosts, 10, 70, 20);
+ order = full(hosts[1], hosts[3], hosts[2]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3])));
+
+ order = full(hosts[4], hosts[1], hosts[3], hosts[2]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3], hosts[4])));
+
+ setScores(dsnitch, 40, allHosts, 10, 10, 11);
+ order = full(hosts[4], hosts[1], hosts[2], hosts[3]);
+ Util.assertRCEquals(order, dsnitch.sortedByProximity(self, full(hosts[1], hosts[2],
hosts[3], hosts[4])));
+ }
+
+ // CASSANDRA-14459 improvements to add latency probes instead of resets
+ @Test
+ public void testLatencyProbeNeeded()
+ {
+ InetAddressAndPort self = hosts[0];
+
+ // Four hosts, two are very latent, but all are requested for ranking
+ dsnitch.receiveTiming(hosts[1], 20, LatencyMeasurementType.READ);
+ dsnitch.receiveTiming(hosts[2], 10, LatencyMeasurementType.READ);
+ dsnitch.receiveTiming(hosts[3], 1000, LatencyMeasurementType.READ);
+ dsnitch.receiveTiming(hosts[4], 1000, LatencyMeasurementType.READ);
+ dsnitch.updateScores();
+
+ EndpointsForRange orderBefore = full(hosts[2], hosts[1], hosts[3], hosts[4]);
+
+ Map<InetAddressAndPort, DynamicEndpointSnitch.AnnotatedMeasurement> measurements
= dsnitch.getMeasurementsWithPort();
+ // At this point we haven't ranked any of hosts so we should need no probes even
+ DynamicEndpointSnitch.calculateProbes(measurements, dsnitch.dynamicLatencyProbeInterval);
+ assertFalse(measurements.values().stream().anyMatch(m -> m.probeTimerMillis
> 0));
+ assertTrue(measurements.values().stream().allMatch(m -> m.probeFuture == null));
+
+ // Two hosts continue receiving traffic but the last two are always ranked by
the snitch
+ for (int i = 0; i < 10; i++)
+ {
+ dsnitch.receiveTiming(hosts[1], 20, LatencyMeasurementType.READ);
+ dsnitch.receiveTiming(hosts[2], 10, LatencyMeasurementType.READ);
+ Util.assertRCEquals(orderBefore, dsnitch.sortedByProximity(self, full(hosts[1],
hosts[2], hosts[3], hosts[4])));
+ }
+ dsnitch.updateScores();
+
+ int probesBefore = (int) ((DynamicEndpointSnitch.MIN_PROBE_INTERVAL_MS / dsnitch.dynamicLatencyProbeInterval)
+ 1);
+
+ for (int i = 0; i < probesBefore; i++)
+ DynamicEndpointSnitch.calculateProbes(measurements, dsnitch.dynamicLatencyProbeInterval);
+ assertEquals(2, measurements.values().stream().filter(m -> m.probeTimerMillis
> 0).count());
+ assertEquals(0, measurements.values().stream().filter(m -> m.probeFuture !=
null).count());
+
+ // Both requested but non measured hosts should have single interval timers set
+ assertEquals(dsnitch.dynamicLatencyProbeInterval, measurements.get(hosts[3]).probeTimerMillis);
+ assertEquals(dsnitch.dynamicLatencyProbeInterval, measurements.get(hosts[4]).probeTimerMillis);
+
+ dsnitch.schedulePings(measurements);
+ // We should have two scheduled futures now
+ assertEquals(2, measurements.values().stream().filter(m -> m.probeFuture !=
null).count());
+
+ int numProbes = 0;
+ for (int i = 1; (1 << i) * dsnitch.dynamicLatencyProbeInterval < DynamicEndpointSnitch.MAX_PROBE_INTERVAL_MS;
i++)
+ {
+ DynamicEndpointSnitch.AnnotatedMeasurement m3 = measurements.get(hosts[3]);
+ DynamicEndpointSnitch.AnnotatedMeasurement m4 = measurements.get(hosts[4]);
+ // Emulate the probe actually running
+ m3.probeFuture.cancel(false);
+ m4.probeFuture.cancel(false);
+ DynamicEndpointSnitch.calculateProbes(measurements, dsnitch.dynamicLatencyProbeInterval);
+ dsnitch.schedulePings(measurements);
+ assertEquals((1 << i) * dsnitch.dynamicLatencyProbeInterval, m3.probeTimerMillis);
+ assertEquals((1 << i) * dsnitch.dynamicLatencyProbeInterval, m4.probeTimerMillis);
+ numProbes++;
+ }
+
+ // If we did exponential backoff right ... we should have the right number of
probes
+ assertTrue(numProbes > 0);
+ assertEquals(numProbes,
+ Math.floor(Math.log(DynamicEndpointSnitch.MAX_PROBE_INTERVAL_MS
/ dsnitch.dynamicLatencyProbeInterval) / Math.log(2.0)),
+ 0.01);
+
+ measurements.get(hosts[3]).probeFuture.cancel(false);
+ measurements.get(hosts[4]).probeFuture.cancel(false);
+
+ // Now we should do ten minute probes until we exceed MAX_PROBE_INTERVAL_MS since
we last requested ranking
+ int interval = dsnitch.dynamicLatencyProbeInterval;
+ for (int i = (probesBefore + numProbes) * interval; i < DynamicEndpointSnitch.MAX_PROBE_INTERVAL_MS;
i += interval)
+ {
+ DynamicEndpointSnitch.calculateProbes(measurements, dsnitch.dynamicLatencyProbeInterval);
+ assertEquals(DynamicEndpointSnitch.MAX_PROBE_INTERVAL_MS, measurements.get(hosts[3]).probeTimerMillis);
+ assertEquals(DynamicEndpointSnitch.MAX_PROBE_INTERVAL_MS, measurements.get(hosts[4]).probeTimerMillis);
+ dsnitch.schedulePings(measurements);
+ measurements.get(hosts[3]).probeFuture.cancel(false);
+ measurements.get(hosts[4]).probeFuture.cancel(false);
+ }
+
+ // Since we haven't measured, we should stop probing
--- End diff --
Correct, typo on my part. Fixed.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: pr-unsubscribe@cassandra.apache.org
For additional commands, e-mail: pr-help@cassandra.apache.org
|