Fixed support for diffing columns of different names (Issue #229)

Author: erezsh

data_diff/diff_tables.py

@@ -172,42 +172,42 @@ def _parse_key_range_result(self, key_type, key_range):
             raise type(e)(f"Cannot apply {key_type} to {mn}, {mx}.") from e
 
     def _validate_and_adjust_columns(self, table1, table2):
-        for c in table1._relevant_columns:
-            if c not in table1._schema:
+        for c1, c2 in safezip(table1._relevant_columns, table2._relevant_columns):
+            if c1 not in table1._schema:
                 raise ValueError(f"Column '{c}' not found in schema for table {table1}")
-            if c not in table2._schema:
+            if c2 not in table2._schema:
                 raise ValueError(f"Column '{c}' not found in schema for table {table2}")
 
             # Update schemas to minimal mutual precision
-            col1 = table1._schema[c]
-            col2 = table2._schema[c]
+            col1 = table1._schema[c1]
+            col2 = table2._schema[c2]
             if isinstance(col1, PrecisionType):
                 if not isinstance(col2, PrecisionType):
-                    raise TypeError(f"Incompatible types for column '{c}':  {col1} <-> {col2}")
+                    raise TypeError(f"Incompatible types for column '{c1}':  {col1} <-> {col2}")
 
                 lowest = min(col1, col2, key=attrgetter("precision"))
 
                 if col1.precision != col2.precision:
-                    logger.warning(f"Using reduced precision {lowest} for column '{c}'. Types={col1}, {col2}")
+                    logger.warning(f"Using reduced precision {lowest} for column '{c1}'. Types={col1}, {col2}")
 
-                table1._schema[c] = col1.replace(precision=lowest.precision, rounds=lowest.rounds)
-                table2._schema[c] = col2.replace(precision=lowest.precision, rounds=lowest.rounds)
+                table1._schema[c1] = col1.replace(precision=lowest.precision, rounds=lowest.rounds)
+                table2._schema[c2] = col2.replace(precision=lowest.precision, rounds=lowest.rounds)
 
             elif isinstance(col1, NumericType):
                 if not isinstance(col2, NumericType):
-                    raise TypeError(f"Incompatible types for column '{c}':  {col1} <-> {col2}")
+                    raise TypeError(f"Incompatible types for column '{c1}':  {col1} <-> {col2}")
 
                 lowest = min(col1, col2, key=attrgetter("precision"))
 
                 if col1.precision != col2.precision:
-                    logger.warning(f"Using reduced precision {lowest} for column '{c}'. Types={col1}, {col2}")
+                    logger.warning(f"Using reduced precision {lowest} for column '{c1}'. Types={col1}, {col2}")
 
-                table1._schema[c] = col1.replace(precision=lowest.precision)
-                table2._schema[c] = col2.replace(precision=lowest.precision)
+                table1._schema[c1] = col1.replace(precision=lowest.precision)
+                table2._schema[c2] = col2.replace(precision=lowest.precision)
 
             elif isinstance(col1, StringType):
                 if not isinstance(col2, StringType):
-                    raise TypeError(f"Incompatible types for column '{c}':  {col1} <-> {col2}")
+                    raise TypeError(f"Incompatible types for column '{c1}':  {col1} <-> {col2}")
 
         for t in [table1, table2]:
             for c in t._relevant_columns:

tests/test_diff_tables.py

@@ -234,25 +234,7 @@ def setUp(self):
             f"create table {self.table_dst}(id int, userid int, movieid int, rating {float_type}, timestamp timestamp)",
             None,
         )
-        # self.preql(
-        #     f"""
-        #     table {self.table_src_name} {{
-        #         userid: int
-        #         movieid: int
-        #         rating: float
-        #         timestamp: timestamp
-        #     }}
-
-        #     table {self.table_dst_name} {{
-        #         userid: int
-        #         movieid: int
-        #         rating: float
-        #         timestamp: timestamp
-        #     }}
-        #     commit()
-        # """
-        # )
-        self.preql.commit()
+        _commit(self.connection)
 
         self.table = TableSegment(self.connection, self.table_src_path, "id", "timestamp", case_sensitive=False)
         self.table2 = TableSegment(self.connection, self.table_dst_path, "id", "timestamp", case_sensitive=False)
@@ -402,6 +384,61 @@ def test_diff_sorted_by_key(self):
         self.assertEqual(expected, diff)
 
 
+@test_per_database
+class TestDiffTables2(TestPerDatabase):
+
+    def test_diff_column_names(self):
+        float_type = _get_float_type(self.connection)
+
+        self.connection.query(
+            f"create table {self.table_src}(id int, rating {float_type}, timestamp timestamp)",
+            None,
+        )
+        self.connection.query(
+            f"create table {self.table_dst}(id2 int, rating2 {float_type}, timestamp2 timestamp)",
+            None,
+        )
+        _commit(self.connection)
+
+        time = "2022-01-01 00:00:00"
+        time2 = "2021-01-01 00:00:00"
+
+        time_str = f"timestamp '{time}'"
+        time_str2 = f"timestamp '{time2}'"
+        _insert_rows(
+            self.connection,
+            self.table_src,
+            ["id", "rating", "timestamp"],
+            [
+                [1, 9, time_str],
+                [2, 9, time_str2],
+                [3, 9, time_str],
+                [4, 9, time_str2],
+                [5, 9, time_str],
+            ],
+        )
+
+        _insert_rows(
+            self.connection,
+            self.table_dst,
+            ["id2", "rating2", "timestamp2"],
+            [
+                [1, 9, time_str],
+                [2, 9, time_str2],
+                [3, 9, time_str],
+                [4, 9, time_str2],
+                [5, 9, time_str],
+            ],
+        )
+
+        table1 = TableSegment(self.connection, self.table_src_path, "id", "timestamp", case_sensitive=False)
+        table2 = TableSegment(self.connection, self.table_dst_path, "id2", "timestamp2", case_sensitive=False)
+
+        differ = TableDiffer()
+        diff = list(differ.diff_tables(table1, table2))
+        assert diff == []
+
+
 @test_per_database
 class TestUUIDs(TestPerDatabase):
     def setUp(self):