Merge branch 'master' into daniel-dx-666-improve-dbt-cloud-event-metadata

Author: dlawin

.github/workflows/label-update_awaiting-response-to-triage.yml

@@ -0,0 +1,23 @@
+# if a comment is added to an issue awaiting response, remove the "awaiting_response" label and add a "triage" label
+
+name: Awaiting Response --> Triage Label
+
+on: issue_comment
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  issues: write
+
+jobs:
+  triage_label:
+    if: contains(github.event.issue.labels.*.name, 'awaiting_response')
+    runs-on: ubuntu-latest
+    steps:
+      - name: initial labeling
+        uses: andymckay/labeler@master
+        with:
+          add-labels: "triage"
+          remove-labels: "awaiting_response"

.github/workflows/label-update_stale-to-triage.yml

@@ -0,0 +1,23 @@
+# if a comment is added to a stale issue, remove the "stale" label and add a "triage" label
+
+name: Stale --> Triage Label
+
+on: issue_comment
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  issues: write
+
+jobs:
+  triage_label:
+    if: contains(github.event.issue.labels.*.name, 'stale')
+    runs-on: ubuntu-latest
+    steps:
+      - name: initial labeling
+        uses: andymckay/labeler@master
+        with:
+          add-labels: "triage"
+          remove-labels: "stale"

.github/workflows/stale.yml

@@ -0,0 +1,29 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+#
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: '28 6 * * *'
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/stale@v5
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue has been marked as stale because it has been open for 60 days with no activity. If you would like the issue to remain open, please comment on the issue and it will be added to the triage queue. Otherwise, it will be closed in 7 days.'
+        stale-pr-message: 'This pull request has been marked as stale because it has been open for 60 days with no activity. If you would like the pull request to remain open, please comment on the pull request and it will be added to the triage queue. Otherwise, it will be closed in 7 days.'
+        stale-issue-label: 'stale'
+        stale-pr-label: 'stale'
+        close-issue-message: "Although we are closing this issue as stale, it's not gone forever. Issues can be reopened if there is renewed community interest. Just add a comment and it will be reopened for triage."
+        close-pr-message: "Although we are closing this pull request as stale, it's not gone forever. PRs can be reopened if there is renewed community interest. Just add a comment and it will be reopened for triage."

.github/workflows/triage_labels.yml

@@ -0,0 +1,21 @@
+name: Label issues
+on:
+  issues:
+    types:
+      - reopened
+      - opened
+jobs:
+  label_issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ["triage"]
+            })

README.md

@@ -2,136 +2,111 @@
     <img alt="Datafold" src="https://user-images.githubusercontent.com/1799931/196497110-d3de1113-a97f-4322-b531-026d859b867a.png" width="50%" />
 </p>
 
-# **data-diff**
+<h1 align="center">
+data-diff
+</h1>
+
+<h2 align="center">
+Develop dbt models faster by testing as you code.
+</h2>
+<h4 align="center">
+See how every change to dbt code affects the data produced in the modified model and downstream.
+</h4>
+<br>
 
 ## What is `data-diff`?
-data-diff is a **free, open-source tool** that enables data professionals to detect differences in values between any two tables.
 
-## Documentation
+data-diff is an open source package that you can use to see the impact of your dbt code changes on your dbt models as you code.
 
-[**🗎 Documentation**](https://docs.datafold.com/guides/os_data_diff) - our detailed documentation has everything you need to start diffing.
+<div align="center">
 
-### Databases we support
+![development_testing_gif](https://user-images.githubusercontent.com/1799931/236354286-d1d044cf-2168-4128-8a21-8c8ca7fd494c.gif)
 
-- PostgreSQL >=10
-- MySQL
-- Snowflake
-- BigQuery
-- Redshift
-- Oracle
-- Presto
-- Databricks
-- Trino
-- Clickhouse
-- Vertica
-- DuckDB >=0.6
-- SQLite (coming soon)
+</div>
 
-For their corresponding connection strings, check out our [detailed table](https://github.com/datafold/data-diff/blob/master/docs/supported-databases.md).
+<br>
 
-#### Looking for a database not on the list?
-If a database is not on the list, we'd still love to support it. [Please open an issue](https://github.com/datafold/data-diff/issues) to discuss it, or vote on existing requests to push them up our todo list.
+:eyes: **Watch 4-min demo video [here](https://www.loom.com/share/ad3df969ba6b4298939efb2fbcc14cde)**
 
-## Get started
+## Getting Started
 
-### Installation
+**Install `data-diff`**
 
-#### First, install `data-diff` using `pip`.
+Install `data-diff` with the command that is specific to the database you use with dbt.
 
+### Snowflake
 ```
-pip install data-diff
+pip install data-diff 'data-diff[snowflake,dbt]' -U
 ```
 
-#### Then, install one or more driver(s) specific to the database(s) you want to connect to.
-
-- `pip install 'data-diff[mysql]'`
-
-- `pip install 'data-diff[postgresql]'`
-
-- `pip install 'data-diff[snowflake]'`
-
-- `pip install 'data-diff[presto]'`
-
-- `pip install 'data-diff[oracle]'`
-
-- `pip install 'data-diff[trino]'`
-
-- `pip install 'data-diff[clickhouse]'`
-
-- `pip install 'data-diff[vertica]'`
-
-- For BigQuery, see: https://pypi.org/project/google-cloud-bigquery/
-
-_Some drivers have dependencies that cannot be installed using `pip` and still need to be installed manually._
-
-### Run your first diff
+### BigQuery
+```
+pip install data-diff 'data-diff[dbt]' google-cloud-bigquery -U
+```
 
-Once you've installed `data-diff`, you can run it from the command line.
+### Redshift
+```
+pip install data-diff 'data-diff[redshift,dbt]' -U
+```
 
+### Postgres
 ```
-data-diff DB1_URI TABLE1_NAME DB2_URI TABLE2_NAME [OPTIONS]
+pip install data-diff 'data-diff[postgres,dbt]' -U
 ```
 
-Be sure to read [the docs](https://docs.datafold.com/reference/open_source/cli) for detailed instructions how to build one of these commands depending on your database setup.
+### Databricks
+```
+pip install data-diff 'data-diff[databricks,dbt]' -U
+```
 
-#### Code Example: Diff Tables Between Databases
-Here's an example command for your copy/pasting, taken from the screenshot above when we diffed data between Snowflake and Postgres.
+### DuckDB
+```
+pip install data-diff 'data-diff[duckdb,dbt]' -U
+```
 
+**Update a few lines in your `dbt_project.yml`**.
 ```
-data-diff \
-  postgresql://<username>:'<password>'@localhost:5432/<database> \
-  <table> \
-  "snowflake://<username>:<password>@<password>/<DATABASE>/<SCHEMA>?warehouse=<WAREHOUSE>&role=<ROLE>" \
-  <TABLE> \
-  -k activity_id \
-  -c activity \
-  -w "event_timestamp < '2022-10-10'"
+#dbt_project.yml
+vars:
+  data_diff:
+    prod_database: my_database
+    prod_schema: my_default_schema
 ```
 
-#### Code Example: Diff Tables Within a Database
+**Run your first data diff!**
 
 ```
-data-diff \
-  "snowflake://<username>:<password>@<password>/<DATABASE>/<SCHEMA_1>?warehouse=<WAREHOUSE>&role=<ROLE>" <TABLE_1> \
-  <SCHEMA_2>.<TABLE_2> \
-  -k org_id \
-  -c created_at -c is_internal \
-  -w "org_id != 1 and org_id < 2000" \
-  -m test_results_%t \
-  --materialize-all-rows \
-  --table-write-limit 10000
+dbt run && data-diff --dbt
 ```
 
-In both code examples, I've used `<>` carrots to represent values that **should be replaced with your values** in the database connection strings. For the flags (`-k`, `-c`, etc.), I opted for "real" values (`org_id`, `is_internal`) to give you a more realistic view of what your command will look like.
+We recommend you get started by walking through [our simple setup instructions](https://docs.datafold.com/development_testing/open_source) which contain examples and details.
+
+Please reach out on the dbt Slack in [#tools-datafold](https://getdbt.slack.com/archives/C03D25A92UU) if you have any trouble whatsoever getting started!
 
-### We're here to help!
+<br><br>
 
-We're here to help! Please post any questions in [GitHub Discussions](https://github.com/datafold/data-diff/discussions).
+### Diffing between databases
 
-## How to Use
+Check out our [documentation](https://docs.datafold.com/reference/open_source/cli) if you're looking to compare data across databases (for example, between Postgres and Snowflake).
 
-* [Examples with dbt, joindiff, and hashdiff](https://docs.datafold.com/reference/open_source/cli#examples)
-* [Examples with Python](https://data-diff.readthedocs.io/en/latest/python-api.html)
-* [How to use with TOML configuration file](https://docs.datafold.com/reference/open_source/cli#toml-config-file)
+<br>
 
-## How to Contribute
-* Feel free to open an issue or contribute to the project by working on an existing issue.
-* Please read the [contributing guidelines](https://github.com/datafold/data-diff/blob/master/CONTRIBUTING.md) to get started.
-* To add a new database driver, check out [docs](https://github.com/datafold/data-diff/blob/master/docs/new-database-driver-guide.rst).
+## Contributors
 
-Big thanks to everyone who contributed so far:
+We thank everyone who contributed so far!
 
 <a href="https://github.com/datafold/data-diff/graphs/contributors">
   <img src="https://contributors-img.web.app/image?repo=datafold/data-diff" />
 </a>
 
-## Technical Explanation
-
-Check out this [technical explanation](https://github.com/datafold/data-diff/blob/master/docs/technical-explanation.md) of how data-diff works.
+<br>
 
 ## Analytics
+
 * [Usage Analytics & Data Privacy](https://github.com/datafold/data-diff/blob/master/docs/usage_analytics.md)
 
+<br>
+
 ## License
 
 This project is licensed under the terms of the [MIT License](https://github.com/datafold/data-diff/blob/master/LICENSE).

data_diff/__main__.py

@@ -228,6 +228,13 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
     metavar="PATH",
     help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.",
 )
+@click.option(
+    "--select",
+    "-s",
+    default=None,
+    metavar="PATH",
+    help="select dbt resources to compare using dbt selection syntax",
+)
 def main(conf, run, **kw):
     if kw["table2"] is None and kw["database2"]:
         # Use the "database table table" form
@@ -264,6 +271,7 @@ def main(conf, run, **kw):
                 profiles_dir_override=kw["dbt_profiles_dir"],
                 project_dir_override=kw["dbt_project_dir"],
                 is_cloud=kw["cloud"],
+                dbt_selection=kw["select"],
             )
         else:
             return _data_diff(**kw)
@@ -306,6 +314,7 @@ def _data_diff(
     cloud,
     dbt_profiles_dir,
     dbt_project_dir,
+    select,
     threads1=None,
     threads2=None,
     __conf__=None,

data_diff/cloud/datafold_api.py

@@ -103,6 +103,8 @@ class TCloudApiDataDiff(pydantic.BaseModel):
     pk_columns: List[str]
     filter1: Optional[str] = None
     filter2: Optional[str] = None
+    include_columns: Optional[List[str]]
+    exclude_columns: Optional[List[str]]
 
 
 class TCloudApiOrgMeta(pydantic.BaseModel):
@@ -138,6 +140,8 @@ class TSummaryResultSchemaStats(pydantic.BaseModel):
     column_type_mismatches: int
     column_reorders: int
     column_counts: Tuple[int, int]
+    column_type_differs: List[str]
+    exclusive_columns: Tuple[List[str], List[str]]
 
 
 class TCloudApiDataDiffSummaryResult(pydantic.BaseModel):
@@ -202,6 +206,11 @@ def get_data_sources(self) -> List[TCloudApiDataSource]:
         rv.raise_for_status()
         return [TCloudApiDataSource(**item) for item in rv.json()]
 
+    def get_data_source(self, data_source_id: int) -> TCloudApiDataSource:
+        rv = self.make_get_request(url=f"api/v1/data_sources/{data_source_id}")
+        rv.raise_for_status()
+        return TCloudApiDataSource(**rv.json())
+
     def create_data_source(self, config: TDsConfig) -> TCloudApiDataSource:
         payload = config.dict()
         if config.type == "bigquery":