fix(floating): floating point numbers as numerics (#1648)

dark0dave · web-flow · commit ecf8598bf66c · 2022-06-07T16:17:43.000-04:00
Signed-off-by: dark0dave &lt;dark0dave@mykolab.com&gt;
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
@@ -296,6 +296,12 @@ private static void fillField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               return;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.setField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              return;
             }
           } else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
             if (val instanceof String) {
@@ -310,6 +316,12 @@ private static void fillField(
                   BigDecimalByteStringEncoder.encodeToBigNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               return;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.setField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToBigNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              return;
             }
           }
         }
@@ -522,6 +534,12 @@ private static void fillRepeatedField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               added = true;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.addRepeatedField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              added = true;
             }
           } else if (fieldSchema != null
               && fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
@@ -537,6 +555,12 @@ private static void fillRepeatedField(
                   BigDecimalByteStringEncoder.encodeToBigNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               added = true;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.addRepeatedField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToBigNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              added = true;
             }
           }
           if (!added) {
diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java
@@ -203,6 +203,12 @@ private static void fillField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               return;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.setField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              return;
             }
           } else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
             if (val instanceof String) {
@@ -217,6 +223,12 @@ private static void fillField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               return;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.setField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              return;
             }
           }
         }
@@ -375,6 +387,12 @@ private static void fillRepeatedField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               added = true;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.addRepeatedField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              added = true;
             }
           } else if (fieldSchema != null
               && fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
@@ -390,6 +408,12 @@ private static void fillRepeatedField(
                   BigDecimalByteStringEncoder.encodeToNumericByteString(
                       new BigDecimal(((Number) val).longValue())));
               added = true;
+            } else if (val instanceof Float || val instanceof Double) {
+              protoMsg.addRepeatedField(
+                  fieldDescriptor,
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(
+                      new BigDecimal(((Number) val).doubleValue())));
+              added = true;
             }
           }
           if (!added) {
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java
@@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
             .setMode(TableFieldSchema.Mode.NULLABLE)
             .setName("test_numeric_long")
             .build();
+    final TableFieldSchema TEST_NUMERIC_FLOAT =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_numeric_float")
+            .build();
+    final TableFieldSchema TEST_NUMERIC_DOUBLE =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_numeric_double")
+            .build();
     final TableFieldSchema TEST_BIGNUMERIC =
         TableFieldSchema.newBuilder()
             .setType(TableFieldSchema.Type.BIGNUMERIC)
@@ -269,6 +281,18 @@ public void testStructComplex() throws Exception {
             .setMode(TableFieldSchema.Mode.NULLABLE)
             .setName("test_bignumeric_long")
             .build();
+    final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.BIGNUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_bignumeric_float")
+            .build();
+    final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.BIGNUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_bignumeric_double")
+            .build();
     final TableFieldSchema TEST_INTERVAL =
         TableFieldSchema.newBuilder()
             .setType(TableFieldSchema.Type.INTERVAL)
@@ -302,12 +326,16 @@ public void testStructComplex() throws Exception {
             .addFields(16, TEST_NUMERIC_STR)
             .addFields(17, TEST_NUMERIC_INT)
             .addFields(18, TEST_NUMERIC_LONG)
-            .addFields(19, TEST_BIGNUMERIC)
-            .addFields(20, TEST_BIGNUMERIC_STR)
-            .addFields(21, TEST_BIGNUMERIC_INT)
-            .addFields(22, TEST_BIGNUMERIC_LONG)
-            .addFields(23, TEST_INTERVAL)
-            .addFields(24, TEST_JSON)
+            .addFields(19, TEST_NUMERIC_FLOAT)
+            .addFields(20, TEST_NUMERIC_DOUBLE)
+            .addFields(21, TEST_BIGNUMERIC)
+            .addFields(22, TEST_BIGNUMERIC_STR)
+            .addFields(23, TEST_BIGNUMERIC_INT)
+            .addFields(24, TEST_BIGNUMERIC_LONG)
+            .addFields(25, TEST_BIGNUMERIC_FLOAT)
+            .addFields(26, TEST_BIGNUMERIC_DOUBLE)
+            .addFields(27, TEST_INTERVAL)
+            .addFields(28, TEST_JSON)
             .build();
     final Descriptor descriptor =
         BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
@@ -421,6 +421,18 @@ public class JsonToProtoMessageTest {
           .setMode(TableFieldSchema.Mode.NULLABLE)
           .setName("test_numeric_long")
           .build();
+  private final TableFieldSchema TEST_NUMERIC_FLOAT =
+      TableFieldSchema.newBuilder()
+          .setType(TableFieldSchema.Type.NUMERIC)
+          .setMode(TableFieldSchema.Mode.NULLABLE)
+          .setName("test_numeric_float")
+          .build();
+  private final TableFieldSchema TEST_NUMERIC_DOUBLE =
+      TableFieldSchema.newBuilder()
+          .setType(TableFieldSchema.Type.NUMERIC)
+          .setMode(TableFieldSchema.Mode.NULLABLE)
+          .setName("test_numeric_double")
+          .build();
   private final TableFieldSchema TEST_BIGNUMERIC =
       TableFieldSchema.newBuilder()
           .setType(TableFieldSchema.Type.BIGNUMERIC)
@@ -445,6 +457,18 @@ public class JsonToProtoMessageTest {
           .setMode(TableFieldSchema.Mode.NULLABLE)
           .setName("test_bignumeric_long")
           .build();
+  private final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
+      TableFieldSchema.newBuilder()
+          .setType(TableFieldSchema.Type.BIGNUMERIC)
+          .setMode(TableFieldSchema.Mode.NULLABLE)
+          .setName("test_bignumeric_float")
+          .build();
+  private final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
+      TableFieldSchema.newBuilder()
+          .setType(TableFieldSchema.Type.BIGNUMERIC)
+          .setMode(TableFieldSchema.Mode.NULLABLE)
+          .setName("test_bignumeric_double")
+          .build();
   final TableFieldSchema TEST_INTERVAL =
       TableFieldSchema.newBuilder()
           .setType(TableFieldSchema.Type.INTERVAL)
@@ -478,12 +502,16 @@ public class JsonToProtoMessageTest {
           .addFields(16, TEST_NUMERIC_STR)
           .addFields(17, TEST_NUMERIC_INT)
           .addFields(18, TEST_NUMERIC_LONG)
-          .addFields(19, TEST_BIGNUMERIC)
-          .addFields(20, TEST_BIGNUMERIC_STR)
-          .addFields(21, TEST_BIGNUMERIC_INT)
-          .addFields(22, TEST_BIGNUMERIC_LONG)
-          .addFields(23, TEST_INTERVAL)
-          .addFields(24, TEST_JSON)
+          .addFields(19, TEST_NUMERIC_FLOAT)
+          .addFields(20, TEST_NUMERIC_DOUBLE)
+          .addFields(21, TEST_BIGNUMERIC)
+          .addFields(22, TEST_BIGNUMERIC_STR)
+          .addFields(23, TEST_BIGNUMERIC_INT)
+          .addFields(24, TEST_BIGNUMERIC_LONG)
+          .addFields(25, TEST_BIGNUMERIC_FLOAT)
+          .addFields(26, TEST_BIGNUMERIC_DOUBLE)
+          .addFields(27, TEST_INTERVAL)
+          .addFields(28, TEST_JSON)
           .build();
 
   @Test
@@ -599,27 +627,6 @@ public void testTimeMismatch() throws Exception {
     }
   }
 
-  @Test
-  public void testNumericMismatch() throws Exception {
-    TableFieldSchema field =
-        TableFieldSchema.newBuilder()
-            .setName("numeric")
-            .setType(TableFieldSchema.Type.NUMERIC)
-            .setMode(TableFieldSchema.Mode.NULLABLE)
-            .build();
-    TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build();
-    JSONObject json = new JSONObject();
-    json.put("numeric", 1.0);
-    try {
-      DynamicMessage protoMsg =
-          JsonToProtoMessage.convertJsonToProtoMessage(
-              TestNumeric.getDescriptor(), tableSchema, json);
-      Assert.fail("should fail");
-    } catch (IllegalArgumentException e) {
-      assertEquals("JSONObject does not have a bytes field at root.numeric.", e.getMessage());
-    }
-  }
-
   @Test
   public void testMixedCaseFieldNames() throws Exception {
     TableFieldSchema field =
@@ -638,27 +645,6 @@ public void testMixedCaseFieldNames() throws Exception {
             TestMixedCaseFieldNames.getDescriptor(), tableSchema, json);
   }
 
-  @Test
-  public void testBigNumericMismatch() throws Exception {
-    TableFieldSchema field =
-        TableFieldSchema.newBuilder()
-            .setName("bignumeric")
-            .setType(TableFieldSchema.Type.BIGNUMERIC)
-            .setMode(TableFieldSchema.Mode.REPEATED)
-            .build();
-    TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build();
-    JSONObject json = new JSONObject();
-    json.put("bignumeric", new JSONArray(new Double[] {1.0}));
-    try {
-      DynamicMessage protoMsg =
-          JsonToProtoMessage.convertJsonToProtoMessage(
-              TestBignumeric.getDescriptor(), tableSchema, json);
-      Assert.fail("should fail");
-    } catch (IllegalArgumentException e) {
-      assertEquals("JSONObject does not have a bytes field at root.bignumeric[0].", e.getMessage());
-    }
-  }
-
   @Test
   public void testDouble() throws Exception {
     TestDouble expectedProto =
@@ -920,6 +906,10 @@ public void testStructComplex() throws Exception {
                 BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1)))
             .setTestNumericLong(
                 BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1L)))
+            .setTestNumericFloat(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1f)))
+            .setTestNumericDouble(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1D)))
             .setTestBignumeric(
                 BigDecimalByteStringEncoder.encodeToBigNumericByteString(
                     new BigDecimal("578960446186580977117854925043439539266.3")))
@@ -929,6 +919,10 @@ public void testStructComplex() throws Exception {
                 BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1)))
             .setTestBignumericLong(
                 BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1L)))
+            .setTestBignumericFloat(
+                BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1f)))
+            .setTestBignumericDouble(
+                BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1D)))
             .setTestInterval("0-0 0 0:0:0.000005")
             .addTestJson("{'a':'b'}")
             .build();
@@ -975,13 +969,17 @@ public void testStructComplex() throws Exception {
     json.put("test_numeric_str", "12.4");
     json.put("test_numeric_int", 1);
     json.put("test_numeric_long", 1L);
+    json.put("test_numeric_float", 1f);
+    json.put("test_numeric_double", 1D);
     json.put(
         "test_bignumeric",
         BigDecimalByteStringEncoder.encodeToBigNumericByteString(
             new BigDecimal("578960446186580977117854925043439539266.3")));
     json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"}));
     json.put("test_bignumeric_int", 1);
     json.put("test_bignumeric_long", 1L);
+    json.put("test_bignumeric_float", 1f);
+    json.put("test_bignumeric_double", 1D);
     json.put("test_interval", "0-0 0 0:0:0.000005");
     json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"}));
     DynamicMessage protoMsg =
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java
@@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
             .setMode(TableFieldSchema.Mode.NULLABLE)
             .setName("test_numeric_long")
             .build();
+    final TableFieldSchema TEST_NUMERIC_FLOAT =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_numeric_float")
+            .build();
+    final TableFieldSchema TEST_NUMERIC_DOUBLE =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_numeric_double")
+            .build();
     final TableFieldSchema TEST_BIGNUMERIC =
         TableFieldSchema.newBuilder()
             .setType(TableFieldSchema.Type.NUMERIC)
@@ -269,6 +281,18 @@ public void testStructComplex() throws Exception {
             .setMode(TableFieldSchema.Mode.NULLABLE)
             .setName("test_bignumeric_long")
             .build();
+    final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_bignumeric_float")
+            .build();
+    final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
+        TableFieldSchema.newBuilder()
+            .setType(TableFieldSchema.Type.NUMERIC)
+            .setMode(TableFieldSchema.Mode.NULLABLE)
+            .setName("test_bignumeric_double")
+            .build();
     final TableFieldSchema TEST_INTERVAL =
         TableFieldSchema.newBuilder()
             .setType(TableFieldSchema.Type.INTERVAL)
@@ -302,12 +326,16 @@ public void testStructComplex() throws Exception {
             .addFields(16, TEST_NUMERIC_STR)
             .addFields(17, TEST_NUMERIC_INT)
             .addFields(18, TEST_NUMERIC_LONG)
-            .addFields(19, TEST_BIGNUMERIC)
-            .addFields(20, TEST_BIGNUMERIC_STR)
-            .addFields(21, TEST_BIGNUMERIC_INT)
-            .addFields(22, TEST_BIGNUMERIC_LONG)
-            .addFields(23, TEST_INTERVAL)
-            .addFields(24, TEST_JSON)
+            .addFields(19, TEST_NUMERIC_FLOAT)
+            .addFields(20, TEST_NUMERIC_DOUBLE)
+            .addFields(21, TEST_BIGNUMERIC)
+            .addFields(22, TEST_BIGNUMERIC_STR)
+            .addFields(23, TEST_BIGNUMERIC_INT)
+            .addFields(24, TEST_BIGNUMERIC_FLOAT)
+            .addFields(25, TEST_BIGNUMERIC_DOUBLE)
+            .addFields(26, TEST_BIGNUMERIC_LONG)
+            .addFields(27, TEST_INTERVAL)
+            .addFields(28, TEST_JSON)
             .build();
     final Descriptor descriptor =
         BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java
diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
