Ad/test

.github/workflows/check-contracts-and-assets.yml

@@ -117,4 +117,20 @@ jobs:
               --proxy-db tutorial \
               --proxy-schema public \
               --proxy-user postgres \
-              --proxy-password postgres
+              --proxy-password postgres
+      - name: Check pyspark
+        uses: gabledata/cicd/github-actions/check-data-assets@latest
+        with:
+            # Provide API key secret, and endpoint variable created in previous steps
+            gable-api-key: ${{secrets.GABLE_API_KEY}}
+            gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT}}
+            allow-gable-pre-release: true
+            gable-version: 0.10.1a9
+            # List of paths to Avro files that should be checked with support for glob syntax. 
+            # Can either be specified as a space separated list ('event1.proto event2.proto'), or 
+            # a multiline string
+            data-asset-options: |
+              --source-type pyspark \
+              --project-root ./pyspark \
+              --spark-job-entrypoint job.py \
+              --csv-schema-file ./pyspark/schemas.csv

.github/workflows/publish-contracts-and-assets.yml

@@ -129,4 +129,4 @@ jobs:
               --proxy-host 0.0.0.0 \
               --proxy-port 5432 \
               --proxy-user postgres \
-              --proxy-password postgres
+              --proxy-password postgres

contracts/pnw_bookings_30_days.yaml

@@ -0,0 +1,45 @@
+id: 16a6daa7-6c17-43b3-bf54-90e11ebcc3f0
+dataAssetResourceName: pyspark://git@github.com:alexdemeo/tutorial:pyspark:job.py:pnw_bookings_30_days
+spec-version: 0.3.0
+doc: bookings for the last 30 days
+name: pnw_bookings_30_days
+namespace: Tutorial
+owner: chad@gable.ai
+schema:
+  - name: booking_date
+    type: string
+    logical: org.iso.8601.DateTime
+    doc: "booking date"
+  - name: commission_rate
+    type: float
+    bits: 64
+    doc: "commission rate"
+  - name: payment_type
+    type: string
+    doc: "payment type"
+  - name: reward_id
+    type: int
+    bits: 64
+    doc: "reward id" 
+  - name: city_id
+    type: string
+    doc: "city id" 
+  - name: city_name
+    type: string
+    doc: "city name" 
+  - name: city_code
+    type: string
+    doc: "city code" 
+  - name: country_id
+    type: string
+    doc: "country id" 
+  - name: region
+    type: string
+    doc: "region" 
+  - name: created_at
+    type: string
+    logical: org.iso.8601.DateTime
+    doc: "created at" 
+  - name: enable_tip
+    type: bool
+    doc: "enable tip" 

pyspark/job.py

@@ -1,20 +1,11 @@
-import argparse
+from script import run_job
+
 from pyspark.sql import SparkSession
 from pyspark.sql.functions import *
 from pyspark.sql.types import *
 
-from script import run_job
-
-def parse_arguments(argv=None):
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--final_output_table")
-    return ap.parse_args(argv)
-
-
 if __name__ == "__main__":
-    # Parse args
-    args_main = parse_arguments()
-    final_output_table = args_main.final_output_table
+    final_output_table = "pnw_bookings_30_days"
     print(f"final_output_table: {final_output_table}")
 
     spark = SparkSession.builder.getOrCreate()

typescript/server/cart.ts

@@ -8,7 +8,7 @@ app.post('/cart', (req, res) => {
   analytics.track({
     userId: req.body.userId,
     event: 'Add to cart',
-    properties: { productId: '123456', quantity: '5' }
+    properties: { /*productId: '123456', */quantity: '5' }
   })
    res.sendStatus(201)
 });