From 547398baa1ddf3bd7461ff3b4cbb7a46de14ba6f Mon Sep 17 00:00:00 2001 From: Florian Mader Date: Wed, 5 Dec 2018 18:14:50 +0100 Subject: [PATCH] added support for datatype Maps --- .../AvroExamples/2-RegisterAssemblies.usql | 4 +- .../AvroExamples/3-SimpleAvro.usql | 4 +- .../AvroExamples/4-EventHubsCapture.usql | 64 +++++++ .../AvroExamples/4-EventHubsCapture.usql.cs | 12 ++ .../AvroExamples/AvroExamples.usqlproj | 10 +- .../AvroExamples/AvroExamples.usqlproj.orig | 39 ++++ .../AvroExtractorTest.cs | 166 +++++++++++++++++- .../Avro/AvroExtractor.cs | 74 +++++++- Examples/Samples/Data/Avro/EHC.avro | Bin 0 -> 70323 bytes Examples/Samples/Data/Avro/EHC.avsc | 30 ++++ docs/index.html | 158 ++++++++--------- 11 files changed, 475 insertions(+), 86 deletions(-) create mode 100644 Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql create mode 100644 Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql.cs create mode 100644 Examples/AvroExamples/AvroExamples/AvroExamples.usqlproj.orig create mode 100644 Examples/Samples/Data/Avro/EHC.avro create mode 100644 Examples/Samples/Data/Avro/EHC.avsc diff --git a/Examples/AvroExamples/AvroExamples/2-RegisterAssemblies.usql b/Examples/AvroExamples/AvroExamples/2-RegisterAssemblies.usql index 864f155..e5cb993 100644 --- a/Examples/AvroExamples/AvroExamples/2-RegisterAssemblies.usql +++ b/Examples/AvroExamples/AvroExamples/2-RegisterAssemblies.usql @@ -1,4 +1,6 @@ -DROP ASSEMBLY IF EXISTS [Avro]; +USE DATABASE [Avro]; + +DROP ASSEMBLY IF EXISTS [Avro]; CREATE ASSEMBLY [Avro] FROM @"/Assemblies/Avro/Avro.dll"; DROP ASSEMBLY IF EXISTS [Microsoft.Analytics.Samples.Formats]; CREATE ASSEMBLY [Microsoft.Analytics.Samples.Formats] FROM @"/Assemblies/Avro/Microsoft.Analytics.Samples.Formats.dll"; diff --git a/Examples/AvroExamples/AvroExamples/3-SimpleAvro.usql b/Examples/AvroExamples/AvroExamples/3-SimpleAvro.usql index c25519c..c81b7e7 100644 --- a/Examples/AvroExamples/AvroExamples/3-SimpleAvro.usql +++ b/Examples/AvroExamples/AvroExamples/3-SimpleAvro.usql @@ -3,8 +3,8 @@ REFERENCE ASSEMBLY [log4net]; REFERENCE ASSEMBLY [Avro]; REFERENCE ASSEMBLY [Microsoft.Analytics.Samples.Formats]; -DECLARE @input_file string = @"\TwitterStream\{*}\{*}\{*}.avro"; -DECLARE @output_file string = @"\output\twitter.csv"; +DECLARE @input_file string = @"/Avro/Samples/twitter.avro"; +DECLARE @output_file string = @"/output/twitter.csv"; @rs = EXTRACT diff --git a/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql b/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql new file mode 100644 index 0000000..251e84f --- /dev/null +++ b/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql @@ -0,0 +1,64 @@ +USE DATABASE Avro; + +REFERENCE ASSEMBLY [Newtonsoft.Json]; +REFERENCE ASSEMBLY [log4net]; +REFERENCE ASSEMBLY [Avro]; +REFERENCE ASSEMBLY [Microsoft.Analytics.Samples.Formats]; + +DECLARE @input_file string = @"/Avro/Samples/EHC.avro"; +DECLARE @output_file string = @"/output/EHC.csv"; + +@rs = + EXTRACT + SequenceNumber long, + Offset string, + EnqueuedTimeUtc string, + SystemProperties SQL.MAP, + Properties SQL.MAP, + Body byte[] + FROM @input_file + USING new Microsoft.Analytics.Samples.Formats.ApacheAvro.AvroExtractor(@" + { + ""type"" : ""record"", + ""name"" : ""EventData"", + ""namespace"" : ""Microsoft.ServiceBus.Messaging"", + ""fields"" : [ { + ""name"" : ""SequenceNumber"", + ""type"" : ""long"" + }, { + ""name"" : ""Offset"", + ""type"" : ""string"" + }, { + ""name"" : ""EnqueuedTimeUtc"", + ""type"" : ""string"" + }, { + ""name"" : ""SystemProperties"", + ""type"" : { + ""type"" : ""map"", + ""values"" : [ ""long"", ""double"", ""string"", ""bytes"" ] + } + }, { + ""name"" : ""Properties"", + ""type"" : { + ""type"" : ""map"", + ""values"" : [ ""long"", ""double"", ""string"", ""bytes"", ""null"" ] + } + }, { + ""name"" : ""Body"", + ""type"" : [ ""null"", ""bytes"" ] + } ] + } + "); + +@mapexplode = + SELECT SequenceNumber, + Offset, + EnqueuedTimeUtc, + Encoding.UTF8.GetString(Body) AS Body, + p.key AS PropertiesKey, + p.value AS PropertiesValue + FROM @rs + OUTER APPLY EXPLODE(Properties) AS p(key, value); + +OUTPUT @mapexplode TO @output_file USING Outputters.Text(); + diff --git a/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql.cs b/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql.cs new file mode 100644 index 0000000..7f89614 --- /dev/null +++ b/Examples/AvroExamples/AvroExamples/4-EventHubsCapture.usql.cs @@ -0,0 +1,12 @@ +using Microsoft.Analytics.Interfaces; +using Microsoft.Analytics.Types.Sql; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace AvroExamples +{ + +} diff --git a/Examples/AvroExamples/AvroExamples/AvroExamples.usqlproj b/Examples/AvroExamples/AvroExamples/AvroExamples.usqlproj index 46ef678..33613a8 100644 --- a/Examples/AvroExamples/AvroExamples/AvroExamples.usqlproj +++ b/Examples/AvroExamples/AvroExamples/AvroExamples.usqlproj @@ -1,6 +1,7 @@  + 2 Debug AnyCPU 2.0 @@ -11,7 +12,6 @@ AvroExamples AvroExamples default - C:\Users\flmader\AppData\Local\USQLDataRoot true @@ -25,6 +25,7 @@