output cols added/removed/types changed

Author: dlawin

data_diff/cloud/datafold_api.py

@@ -132,6 +132,8 @@ class TSummaryResultSchemaStats(pydantic.BaseModel):
     column_type_mismatches: int
     column_reorders: int
     column_counts: Tuple[int, int]
+    column_type_differs: List[str]
+    exclusive_columns: Tuple[List[str], List[str]]
 
 
 class TCloudApiDataDiffSummaryResult(pydantic.BaseModel):

data_diff/dbt.py

@@ -6,7 +6,13 @@
 
 from dataclasses import dataclass
 from typing import List, Optional, Dict
-from .utils import dbt_diff_string_template, getLogger
+from .utils import (
+    dbt_diff_string_template,
+    getLogger,
+    columns_added_template,
+    columns_removed_template,
+    no_differences_template,
+)
 from pathlib import Path
 
 import keyring
@@ -182,7 +188,6 @@ def _get_diff_vars(
 
 
 def _local_diff(diff_vars: DiffVars) -> None:
-    column_diffs_str = ""
     dev_qualified_str = ".".join(diff_vars.dev_path)
     prod_qualified_str = ".".join(diff_vars.prod_path)
     diff_output_str = _diff_output_base(dev_qualified_str, prod_qualified_str)
@@ -203,14 +208,14 @@ def _local_diff(diff_vars: DiffVars) -> None:
         return
 
     mutual_set = set(table1_columns) & set(table2_columns)
-    table1_set_diff = list(set(table1_columns) - set(table2_columns))
-    table2_set_diff = list(set(table2_columns) - set(table1_columns))
+    columns_added = list(set(table1_columns) - set(table2_columns))
+    columns_removed = list(set(table2_columns) - set(table1_columns))
 
-    if table1_set_diff:
-        column_diffs_str += "Column(s) added: " + str(table1_set_diff) + "\n"
+    if columns_added:
+        diff_output_str += columns_added_template(columns_added)
 
-    if table2_set_diff:
-        column_diffs_str += "Column(s) removed: " + str(table2_set_diff) + "\n"
+    if columns_removed:
+        diff_output_str += columns_removed_template(columns_removed)
 
     mutual_set = mutual_set - set(diff_vars.primary_keys)
     extra_columns = tuple(mutual_set)
@@ -225,10 +230,10 @@ def _local_diff(diff_vars: DiffVars) -> None:
     )
 
     if list(diff):
-        diff_output_str += f"{column_diffs_str}{diff.get_stats_string(is_dbt=True)} \n"
+        diff_output_str += f"{diff.get_stats_string(is_dbt=True)} \n"
         rich.print(diff_output_str)
     else:
-        diff_output_str += f"{column_diffs_str}[bold][green]No row differences[/][/] \n"
+        diff_output_str += no_differences_template()
         rich.print(diff_output_str)
 
 
@@ -302,6 +307,18 @@ def _cloud_diff(diff_vars: DiffVars, datasource_id: int, api: DatafoldAPI) -> No
         diff_percent_list = {
             x.column_name: str(x.match) + "%" for x in diff_results.values.columns_diff_stats if x.match != 100.0
         }
+        columns_added = diff_results.schema_.exclusive_columns[1]
+        columns_removed = diff_results.schema_.exclusive_columns[0]
+        column_type_changes = diff_results.schema_.column_type_differs
+
+        if columns_added:
+            diff_output_str += columns_added_template(columns_added)
+
+        if columns_removed:
+            diff_output_str += columns_removed_template(columns_removed)
+
+        if column_type_changes:
+            diff_output_str += "Type change: " + str(column_type_changes) + "\n"
 
         if any([rows_added_count, rows_removed_count, rows_updated]):
             diff_output = dbt_diff_string_template(
@@ -312,10 +329,10 @@ def _cloud_diff(diff_vars: DiffVars, datasource_id: int, api: DatafoldAPI) -> No
                 diff_percent_list,
                 "Value Match Percent:",
             )
-            diff_output_str += f"{diff_url}\n {diff_output} \n"
+            diff_output_str += f"\n{diff_url}\n {diff_output} \n"
             rich.print(diff_output_str)
         else:
-            diff_output_str += f"{diff_url}\n [green]No row differences[/] \n"
+            diff_output_str += f"\n{diff_url}\n{no_differences_template()}\n"
             rich.print(diff_output_str)
 
     except BaseException as ex:  # Catch KeyboardInterrupt too

data_diff/utils.py

@@ -144,3 +144,17 @@ def dbt_diff_string_template(
         string_output += f"\n{k}: {v}"
 
     return string_output
+
+
+def columns_removed_template(table2_set_diff) -> str:
+    columns_removed = "Column(s) removed: " + str(table2_set_diff) + "\n"
+    return columns_removed
+
+
+def columns_added_template(table1_set_diff) -> str:
+    columns_added = "Column(s) added: " + str(table1_set_diff) + "\n"
+    return columns_added
+
+
+def no_differences_template() -> str:
+    return "[bold][green]No row differences[/][/]\n"