avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject svn commit: r1492821 - in /avro/trunk: CHANGES.txt lang/c++/impl/json/JsonIO.cc lang/c++/impl/json/JsonIO.hh lang/c++/impl/parsing/JsonCodec.cc lang/c++/test/CodecTests.cc
Date Thu, 13 Jun 2013 19:02:46 GMT
Author: thiru
Date: Thu Jun 13 19:02:46 2013
New Revision: 1492821

URL: http://svn.apache.org/r1492821
Log:
AVRO-1290. Handling NaN and positive and negative infinities in C++ Json

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/c++/impl/json/JsonIO.cc
    avro/trunk/lang/c++/impl/json/JsonIO.hh
    avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
    avro/trunk/lang/c++/test/CodecTests.cc

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jun 13 19:02:46 2013
@@ -62,6 +62,8 @@ Trunk (not yet released)
     AVRO-1324. C: Handle namespaces in schema parsing.
     (Ben Walsh via dcreager)
 
+    AVRO-1290. Handling NaN and positive and negative infinities in C++ Json (Daniel Russel
via thiru)
+
   BUG FIXES
 
     AVRO-1296. Python: Fix schemas retrieved from protocol types

Modified: avro/trunk/lang/c++/impl/json/JsonIO.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.cc (original)
+++ avro/trunk/lang/c++/impl/json/JsonIO.cc Thu Jun 13 19:02:46 2013
@@ -21,6 +21,8 @@
 namespace avro {
 namespace json {
 
+using std::ostringstream;
+
 const char* const
 JsonParser::tokenNames[] = {
     "Null",
@@ -44,6 +46,25 @@ char JsonParser::next()
     return ch;
 }
 
+void JsonParser::expectToken(Token tk)
+{
+    if (advance() != tk) {
+        if (tk == tkDouble && cur() == tkString
+            && (sv == "Infinity" || sv == "-Infinity" || sv == "NaN")) {
+            curToken = tkDouble;
+            dv = sv == "Infinity" ? std::numeric_limits<double>::infinity() :
+                sv == "-Infinity" ? -std::numeric_limits<double>::infinity() :
+                std::numeric_limits<double>::quiet_NaN();
+            return;
+        }
+        ostringstream oss;
+        oss << "Incorrect token in the stream. Expected: "
+            << JsonParser::toString(tk) << ", found "
+            << JsonParser::toString(cur());
+        throw Exception(oss.str());
+    }
+}
+
 JsonParser::Token JsonParser::doAdvance()
 {
     char ch = next();
@@ -122,14 +143,14 @@ JsonParser::Token JsonParser::tryNumber(
     sv.push_back(ch);
 
     hasNext = false;
-    int state = (ch == '-') ? 0 : (ch == 0) ? 1 : 2;
+    int state = (ch == '-') ? 0 : (ch == '0') ? 1 : 2;
     for (; ;) {
         switch (state) {
         case 0:
             if (in_.hasMore()) {
                 ch = in_.read();
                 if (isdigit(ch)) {
-                    state = (ch == 0) ? 1 : 2;
+                    state = (ch == '0') ? 1 : 2;
                     sv.push_back(ch);
                     continue;
                 }

Modified: avro/trunk/lang/c++/impl/json/JsonIO.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/json/JsonIO.hh?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/json/JsonIO.hh (original)
+++ avro/trunk/lang/c++/impl/json/JsonIO.hh Thu Jun 13 19:02:46 2013
@@ -102,6 +102,8 @@ public:
         return curToken;
     }
 
+    void expectToken(Token tk);
+
     bool boolValue() {
         return bv;
     }

Modified: avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/parsing/JsonCodec.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/parsing/JsonCodec.cc (original)
+++ avro/trunk/lang/c++/impl/parsing/JsonCodec.cc Thu Jun 13 19:02:46 2013
@@ -26,6 +26,7 @@
 #include <boost/make_shared.hpp>
 #include <boost/weak_ptr.hpp>
 #include <boost/any.hpp>
+#include <boost/math/special_functions/fpclassify.hpp>
 
 #include "ValidatingCodec.hh"
 #include "Symbol.hh"
@@ -160,13 +161,7 @@ Production JsonGrammarGenerator::doGener
 
 static void expectToken(JsonParser& in, JsonParser::Token tk)
 {
-    if (in.advance() != tk) {
-        ostringstream oss;
-        oss << "Incorrect token in the stream. Expected: "
-            << JsonParser::toString(tk) << ", found "
-            << JsonParser::toString(in.cur());
-        throw Exception(oss.str());
-    }
+    in.expectToken(tk);
 }
 
 class JsonDecoderHandler {
@@ -573,14 +568,30 @@ template<typename P>
 void JsonEncoder<P>::encodeFloat(float f)
 {
     parser_.advance(Symbol::sFloat);
-    out_.encodeNumber(f);
+    if (f == std::numeric_limits<float>::infinity()) {
+        out_.encodeString("Infinity");
+    } else if (f == -std::numeric_limits<float>::infinity()) {
+        out_.encodeString("-Infinity");
+    } else if (boost::math::isnan(f)) {
+        out_.encodeString("NaN");
+    } else {
+        out_.encodeNumber(f);
+    }
 }
 
 template<typename P>
 void JsonEncoder<P>::encodeDouble(double d)
 {
     parser_.advance(Symbol::sDouble);
-    out_.encodeNumber(d);
+    if (d == std::numeric_limits<double>::infinity()) {
+        out_.encodeString("Infinity");
+    } else if (d == -std::numeric_limits<double>::infinity()) {
+        out_.encodeString("-Infinity");
+    } else if (boost::math::isnan(d)) {
+        out_.encodeString("NaN");
+    } else {
+        out_.encodeNumber(d);
+    }
 }
 
 template<typename P>

Modified: avro/trunk/lang/c++/test/CodecTests.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/test/CodecTests.cc?rev=1492821&r1=1492820&r2=1492821&view=diff
==============================================================================
--- avro/trunk/lang/c++/test/CodecTests.cc (original)
+++ avro/trunk/lang/c++/test/CodecTests.cc Thu Jun 13 19:02:46 2013
@@ -36,8 +36,24 @@
 #include <boost/test/unit_test.hpp>
 #include <boost/test/parameterized_test.hpp>
 #include <boost/random/mersenne_twister.hpp>
+#include <boost/math/special_functions/fpclassify.hpp>
 
 namespace avro {
+
+/*
+void dump(const OutputStream& os)
+{
+    std::auto_ptr<InputStream> in = memoryInputStream(os);
+    const char *b;
+    size_t n;
+    std::cout << os.byteCount() << std::endl;
+    while (in->next(reinterpret_cast<const uint8_t**>(&b), &n)) {
+        std::cout << std::string(b, n);
+    }
+    std::cout << std::endl;
+}
+*/
+
 namespace parsing {
 
 static const unsigned int count = 10;
@@ -589,20 +605,6 @@ struct TestData4 {
     unsigned int depth;
 };
 
-/*
-static void dump(const OutputStream& os)
-{
-    std::auto_ptr<InputStream> in = memoryInputStream(os);
-    const char *b;
-    size_t n;
-    std::cout << os.byteCount() << std::endl;
-    while (in->next(reinterpret_cast<const uint8_t**>(&b), &n)) {
-        std::cout << std::string(b, n);
-    }
-    std::cout << std::endl;
-}
-*/
-
 template<typename CodecFactory>
 void testCodec(const TestData& td) {
     static int testNo = 0;
@@ -1418,6 +1420,57 @@ static void testStreamLifetimes()
 
 }
 
+static void testLimits(const EncoderPtr& e, const DecoderPtr& d)
+{
+    std::auto_ptr<OutputStream> s1 = memoryOutputStream();
+    {
+        e->init(*s1);
+        e->encodeDouble(std::numeric_limits<double>::infinity());
+        e->encodeDouble(-std::numeric_limits<double>::infinity());
+        e->encodeDouble(std::numeric_limits<double>::quiet_NaN());
+        e->encodeFloat(std::numeric_limits<float>::infinity());
+        e->encodeFloat(-std::numeric_limits<float>::infinity());
+        e->encodeFloat(std::numeric_limits<float>::quiet_NaN());
+        e->flush();
+    }
+
+    {
+        std::auto_ptr<InputStream> s2 = memoryInputStream(*s1);
+        d->init(*s2);
+        BOOST_CHECK_EQUAL(d->decodeDouble(),
+            std::numeric_limits<double>::infinity());
+        BOOST_CHECK_EQUAL(d->decodeDouble(),
+            -std::numeric_limits<double>::infinity());
+        BOOST_CHECK(boost::math::isnan(d->decodeDouble()));
+        BOOST_CHECK_EQUAL(d->decodeFloat(),
+            std::numeric_limits<float>::infinity());
+        BOOST_CHECK_EQUAL(d->decodeFloat(),
+            -std::numeric_limits<float>::infinity());
+        BOOST_CHECK(boost::math::isnan(d->decodeFloat()));
+    }
+
+}
+
+static void testLimitsBinaryCodec()
+{
+    testLimits(binaryEncoder(), binaryDecoder());
+}
+
+static void testLimitsJsonCodec()
+{
+    const char* s = "{ \"type\": \"record\", \"name\": \"r\", \"fields\": ["
+        "{ \"name\": \"d1\", \"type\": \"double\" },"
+        "{ \"name\": \"d2\", \"type\": \"double\" },"
+        "{ \"name\": \"d3\", \"type\": \"double\" },"
+        "{ \"name\": \"f1\", \"type\": \"float\" },"
+        "{ \"name\": \"f2\", \"type\": \"float\" },"
+        "{ \"name\": \"f3\", \"type\": \"float\" }"
+    "]}";
+    ValidSchema schema = parsing::makeValidSchema(s);
+    testLimits(jsonEncoder(schema), jsonDecoder(schema));
+}
+
+
 }   // namespace avro
 
 boost::unit_test::test_suite*
@@ -1428,6 +1481,8 @@ init_unit_test_suite( int argc, char* ar
     test_suite* ts= BOOST_TEST_SUITE("Avro C++ unit tests for codecs");
     avro::parsing::add_tests(*ts);
     ts->add(BOOST_TEST_CASE(avro::testStreamLifetimes));
+    ts->add(BOOST_TEST_CASE(avro::testLimitsBinaryCodec));
+    ts->add(BOOST_TEST_CASE(avro::testLimitsJsonCodec));
 
     return ts;
 }



Mime
View raw message