From 6b42732627dba0a5bf0189dbfcb27aff0fdd2f56 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 17 Dec 2018 13:51:36 +0100 Subject: [PATCH 1/5] updated gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0c01640..20bfa33 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,9 @@ GeneratedArtifacts/ _Pvt_Extensions/ ModelManifest.xml +# Visual Studio 2015/2017 cache/options directory +.vs/ + # ========================= # Windows detritus # ========================= From cc69e94acad2f0ed044497413fa74d70e04bd3d4 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 17 Dec 2018 13:07:08 +0100 Subject: [PATCH 2/5] added multiline json outputter --- .../JsonLinesOutputterTests.cs | 687 ++++++++++++++++++ ...oft.Analytics.Samples.Formats.Tests.csproj | 1 + .../Json/JsonLinesOutputter.cs | 189 +++++ ...Microsoft.Analytics.Samples.Formats.csproj | 1 + 4 files changed, 878 insertions(+) create mode 100644 Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs create mode 100644 Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs new file mode 100644 index 0000000..3305579 --- /dev/null +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs @@ -0,0 +1,687 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.Analytics.Samples.Formats.Json; +using System.IO; +using Microsoft.Analytics.UnitTest; +using Microsoft.Analytics.Interfaces; +using System.Collections.Generic; +using System.Text; +using Microsoft.Analytics.Types.Sql; + +namespace Microsoft.Analytics.Samples.Formats.Tests +{ + [TestClass] + public class JsonLinesOutputterTests + { + [TestMethod] + public void JsonOutputter_DatatypeShort_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + short a = 0; + short b = 1; + object[] values = new object[2] { a, b }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":0,\"b\":1}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableShort_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + short a = 0; + short b = 1; + object[] values = new object[3] { a, b, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":0,\"b\":1}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeInt_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 0, 1 }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":0,\"b\":1}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableInt_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 0, 1, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":0,\"b\":1}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeLong_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 9223372036854775807, -9223372036854775807 }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":9223372036854775807,\"b\":-9223372036854775807}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableLong_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 9223372036854775807, -9223372036854775807, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":9223372036854775807,\"b\":-9223372036854775807}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeFloat_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 3.5F, 0F }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":3.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableFloat_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 3.5F, 0F, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":3.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeDouble_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 3.5D, 0D }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":3.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableDouble_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 3.5, 0D, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":3.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeDecimal_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 350.5M, 0M }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":350.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableDecimal_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 350.5M, 0M, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":350.5,\"b\":0.0}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeByte_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + + byte a = 2; + byte b = 4; + + object[] values = new object[2] { a, b }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":2,\"b\":4}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableBytes_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + + byte? a = 2; + byte? b = null; + + object[] values = new object[2] { a, b }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":2}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeBoolean_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { true, false }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":true,\"b\":false}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableBoolean_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { true, false, null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":true,\"b\":false}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeString_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { "test", "", null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":\"test\",\"b\":\"\"}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeChar_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { 'a', ' ' }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":\"a\",\"b\":\" \"}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableChar_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c") + ); + object[] values = new object[3] { 'a', ' ', null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":\"a\",\"b\":\" \"}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeDateTime_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a") + ); + object[] values = new object[1] { new DateTime(2010, 01, 05) }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":\"2010-01-05T00:00:00\"}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeNullableDateTime_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b") + ); + object[] values = new object[2] { new DateTime(2010, 01, 05), null }; + var row = new USqlRow(schema, values); + + var expected = "{\"a\":\"2010-01-05T00:00:00\"}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeArrayOfPrimitiveTypes_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn("a"), + new USqlColumn("b"), + new USqlColumn("c"), + new USqlColumn("d"), + new USqlColumn("e"), + new USqlColumn("f"), + new USqlColumn("g"), + new USqlColumn("h"), + new USqlColumn("i"), + new USqlColumn("j"), + new USqlColumn("k") + ); + object[] values = new object[11] { + new short?[3] { 0, 1, null }, + new int?[3] { 0, 1, null }, + new long?[3] { 9223372036854775807, -9223372036854775807, null }, + new float?[3] { 3.5F, 0F, null }, + new double?[3] { 3.5D, 0D, null }, + new decimal?[3] { 205.2M, 0M, null }, + new byte?[3] { 2, 4, null }, + new bool?[3] { true, false, null }, + new string[3] { "test", "", null }, + new char?[3] { 'a', ' ', null }, + new DateTime?[3] { new DateTime(2010, 01, 05), new DateTime(2015, 05, 06), null } + }; + + var row = new USqlRow(schema, values); + + var expected = "{" + + "\"a\":[0,1]," + + "\"b\":[0,1]," + + "\"c\":[9223372036854775807,-9223372036854775807]," + + "\"d\":[3.5,0.0]," + + "\"e\":[3.5,0.0]," + + "\"f\":[205.2,0.0]," + + "\"g\":[2,4]," + + "\"h\":[true,false]," + + "\"i\":[\"test\",\"\"]," + + "\"j\":[\"a\",\" \"]," + + "\"k\":[\"2010-01-05T00:00:00\",\"2015-05-06T00:00:00\"]" + + "}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeDictionaryOfPrimitiveTypes_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + var dict = new Dictionary + { + { "short", (short)3 }, + { "int", 3 }, + { "long", 9223372036854775807 }, + { "float", 3.5F }, + { "double", 3.5D }, + { "decimal", 205.2M }, + { "byte", (byte)3 }, + { "bool", true }, + { "string", "test" }, + { "char", 'a' }, + { "DateTime", new DateTime(2015, 05, 06) } + }; + + object[] values = new object[1] { dict }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":{" + + "\"short\":3," + + "\"int\":3," + + "\"long\":9223372036854775807," + + "\"float\":3.5," + + "\"double\":3.5," + + "\"decimal\":205.2," + + "\"byte\":3," + + "\"bool\":true," + + "\"string\":\"test\"," + + "\"char\":\"a\"," + + "\"DateTime\":\"2015-05-06T00:00:00\"" + + "}}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeSqlMapOfPrimitiveTypes_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + var map = new SqlMap(new Dictionary + { + { "short", (short)3 }, + { "int", 3 }, + { "long", 9223372036854775807 }, + { "float", 3.5F }, + { "double", 3.5D }, + { "decimal", 205.2M }, + { "byte", (byte)3 }, + { "bool", true }, + { "string", "test" }, + { "char", 'a' }, + { "DateTime", new DateTime(2015, 05, 06) } + }); + + object[] values = new object[1] { map }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":{" + + "\"short\":3," + + "\"int\":3," + + "\"long\":9223372036854775807," + + "\"float\":3.5," + + "\"double\":3.5," + + "\"decimal\":205.2," + + "\"byte\":3," + + "\"bool\":true," + + "\"string\":\"test\"," + + "\"char\":\"a\"," + + "\"DateTime\":\"2015-05-06T00:00:00\"" + + "}}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeKeyValuePairPrimitiveTypes_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + var keyValuePair = new KeyValuePair("int", 3); + + object[] values = new object[1] { keyValuePair }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":{\"int\":3}}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeArrayOfMaps_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn[]>("a") + ); + + Dictionary[] dictArray = new Dictionary[2] + { + new Dictionary + { + { "test1", "asd" }, + { "test2", 3 }, + }, + new Dictionary + { + { "test1", "das" }, + { "test2", 3 }, + } + }; + + object[] values = new object[1] { dictArray }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":[{\"test1\":\"asd\",\"test2\":3},{\"test1\":\"das\",\"test2\":3}]}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeMapOfArrays_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + + Dictionary dict = new Dictionary + { + { "test1", new int[]{ 2, 3 } }, + { "test2", new string[]{ "asd", "" } }, + }; + + object[] values = new object[1] { dict }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":{\"test1\":[2,3],\"test2\":[\"asd\",\"\"]}}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeMapOfArrays_MultiRow_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + + List rows = new List(); + Dictionary dict = new Dictionary + { + { "test1", new int[]{ 2, 3 } }, + { "test2", new string[]{ "asd", "" } }, + }; + + rows.Add(new USqlRow(schema, new object[] { dict })); + + dict = new Dictionary + { + { "test3", new int[]{ 1, 4 } }, + { "test4", new string[]{ "foo", "bar" } }, + }; + + rows.Add(new USqlRow(schema, new object[] { dict })); + + + var expected = "{\"a\":{\"test1\":[2,3],\"test2\":[\"asd\",\"\"]}}"+ + Environment.NewLine + + "{\"a\":{\"test3\":[1,4],\"test4\":[\"foo\",\"bar\"]}}"; + var actual = GetOutputterResult(rows); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void JsonOutputter_DatatypeComplex_Outputted() + { + USqlSchema schema = new USqlSchema( + new USqlColumn>("a") + ); + + Dictionary complex = new Dictionary + { + { + "test1", new Dictionary(){ { "nested", 1}, { "nestedArray", new int[] { 1, 2 } } } + }, + { + "test2", new Dictionary[] + { + new Dictionary + { + { "test1", "asd" }, + { "test2", 3 }, + }, + new Dictionary + { + { "test1", "das" }, + { "test2", 3 }, + } + } + } + }; + + object[] values = new object[1] { complex }; + + var row = new USqlRow(schema, values); + + var expected = "{\"a\":" + + "{" + + "\"test1\":" + + "{" + + "\"nested\":1," + + "\"nestedArray\":[1,2]" + + "}," + + "\"test2\":" + + "[" + + "{\"test1\":\"asd\",\"test2\":3}," + + "{\"test1\":\"das\",\"test2\":3}" + + "]" + + "}" + + "}"; + var actual = GetOutputterResult(row); + + Assert.AreEqual(expected, actual); + } + + private string GetOutputterResult(IEnumerable rows) + { + var outputter = new JsonLinesOutputter(); + + using (var ms = new MemoryStream()) + { + var unstructuredWriter = new USqlStreamWriter(ms); + foreach (IRow row in rows) + { + outputter.Output(row, unstructuredWriter); + } + outputter.Close(); + var output = Encoding.ASCII.GetString(ms.ToArray()).TrimEnd(); + return output; + } + } + + private string GetOutputterResult(IRow row) + { + return GetOutputterResult(new[] { row }); + } + } +} diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/Microsoft.Analytics.Samples.Formats.Tests.csproj b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/Microsoft.Analytics.Samples.Formats.Tests.csproj index fa6a760..ea88d43 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/Microsoft.Analytics.Samples.Formats.Tests.csproj +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/Microsoft.Analytics.Samples.Formats.Tests.csproj @@ -67,6 +67,7 @@ + diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs new file mode 100644 index 0000000..e9893cd --- /dev/null +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs @@ -0,0 +1,189 @@ +// +// Copyright (c) Microsoft and contributors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Collections; +using System.IO; +using Microsoft.Analytics.Interfaces; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Microsoft.Analytics.Types.Sql; + +namespace Microsoft.Analytics.Samples.Formats.Json +{ + /// + /// JsonLinesOutputter (sample) + /// + /// IEnumerable[IRow] => + /// { c1:r1v1, c2:r1v2, ...} + /// { c1:r2v2, c2:r2v2, ...} + /// ... + /// Notice that this outputter doesn't require atomic output, + /// since it produces standalone JSON documents as opposed to a single JSON array. + /// + [SqlUserDefinedOutputter(AtomicFileProcessing = false)] + public class JsonLinesOutputter : IOutputter + { + /// + private JsonTextWriter writer; + + private static readonly KeyValuePair KVP = new KeyValuePair(string.Empty,string.Empty); + private const string keyPropName = nameof(KVP.Key); + private const string valPropName = nameof(KVP.Value); + + /// + public JsonLinesOutputter() + { + } + + /// + public override void Output(IRow row, IUnstructuredWriter output) + { + if (this.writer == null) + { + // Json.Net (writer) + this.writer = new JsonTextWriter(new StreamWriter(output.BaseStream)) { Formatting = Formatting.None }; + } + + // Row(s) + WriteRow(row, this.writer); + } + + /// + public override void Close() + { + if (this.writer != null) + { + this.writer.Flush(); + this.writer.Close(); + } + } + + /// + private static void WriteRow(IRow row, JsonTextWriter writer) + { + // Row + // => { c1:v1, c2:v2, ...} + + // Header + writer.WriteStartObject(); + + // Fields + var columns = row.Schema; + for (int i = 0; i < columns.Count; i++) + { + // Note: We simply delegate to Json.Net for all data conversions + // For data conversions beyond what Json.Net supports, do an explicit projection: + // ie: SELECT datetime.ToString(...) AS datetime, ... + object value = row.Get(i); + + // Note: We don't bloat the JSON with sparse (null) properties + if (value != null) + { + writer.WritePropertyName(columns[i].Name, escape: true); + WriteValue(writer, value); + } + } + + // Footer + writer.WriteEndObject(); + writer.WriteWhitespace(Environment.NewLine); + } + + [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")] + private static void WriteValue(JsonTextWriter writer, object value) + { + if (value != null) + { + IEnumerable collection = value as IEnumerable; + Type valueType = value.GetType(); + + if (IsArray(collection)) + { + // Dictionary + if (IsMap(valueType)) + { + WriteMapAsEnumerable(writer, collection); + } + // Array + else + { + WriteArray(writer, collection); + } + } + // KeyValue + else if (IsMap(valueType)) + { + WriteKeyValuePair(writer, value); + } + else + writer.WriteValue(value); + } + } + + private static void WriteKeyValuePair(JsonTextWriter writer, object kvp) + { + + WriteMapAsEnumerable(writer, new[] { kvp }); + } + + private static void WriteArray(JsonTextWriter writer, IEnumerable collection) + { + writer.WriteStartArray(); + foreach (var item in collection) + { + WriteValue(writer, item); + } + writer.WriteEndArray(); + } + + private static void WriteMapAsEnumerable(JsonTextWriter writer, IEnumerable collection) + { + writer.WriteStartObject(); + PropertyInfo keyProp = null; + PropertyInfo valProp = null; + foreach (var item in collection) + { + if (keyProp == null) + { + Type itemType = item.GetType(); + keyProp = itemType.GetProperty(keyPropName); + valProp = itemType.GetProperty(valPropName); + } + // ReSharper disable once PossibleNullReferenceException + writer.WritePropertyName(keyProp.GetValue(item, null).ToString(), escape: true); + // ReSharper disable once PossibleNullReferenceException + WriteValue(writer, valProp.GetValue(item, null)); + } + writer.WriteEndObject(); + } + + private static bool IsArray(IEnumerable collection) + { + return collection != null && !(collection is string); + } + + private static bool IsMap(Type valueType) + { + return valueType.IsGenericType && + (valueType.GetGenericTypeDefinition() == typeof(KeyValuePair<,>) || + valueType.GetGenericTypeDefinition() == typeof(Dictionary<,>) || + valueType.GetGenericTypeDefinition() == typeof(SqlMap<,>)); + } + } +} diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Microsoft.Analytics.Samples.Formats.csproj b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Microsoft.Analytics.Samples.Formats.csproj index a0f1d46..2ea1b70 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Microsoft.Analytics.Samples.Formats.csproj +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Microsoft.Analytics.Samples.Formats.csproj @@ -53,6 +53,7 @@ + From a4b27f0df7ad2abfd271dc2ccceb3b450ef7bd37 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 17 Dec 2018 14:20:06 +0100 Subject: [PATCH 3/5] code cleanup --- .../JsonLinesOutputterTests.cs | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs index 3305579..8723df3 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs @@ -22,7 +22,7 @@ public void JsonOutputter_DatatypeShort_Outputted() ); short a = 0; short b = 1; - object[] values = new object[2] { a, b }; + object[] values = { a, b }; var row = new USqlRow(schema, values); var expected = "{\"a\":0,\"b\":1}"; @@ -41,7 +41,7 @@ public void JsonOutputter_DatatypeNullableShort_Outputted() ); short a = 0; short b = 1; - object[] values = new object[3] { a, b, null }; + object[] values = { a, b, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":0,\"b\":1}"; @@ -57,7 +57,7 @@ public void JsonOutputter_DatatypeInt_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 0, 1 }; + object[] values = { 0, 1 }; var row = new USqlRow(schema, values); var expected = "{\"a\":0,\"b\":1}"; @@ -74,7 +74,7 @@ public void JsonOutputter_DatatypeNullableInt_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 0, 1, null }; + object[] values = { 0, 1, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":0,\"b\":1}"; @@ -90,7 +90,7 @@ public void JsonOutputter_DatatypeLong_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 9223372036854775807, -9223372036854775807 }; + object[] values = { 9223372036854775807, -9223372036854775807 }; var row = new USqlRow(schema, values); var expected = "{\"a\":9223372036854775807,\"b\":-9223372036854775807}"; @@ -107,7 +107,7 @@ public void JsonOutputter_DatatypeNullableLong_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 9223372036854775807, -9223372036854775807, null }; + object[] values = { 9223372036854775807, -9223372036854775807, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":9223372036854775807,\"b\":-9223372036854775807}"; @@ -123,7 +123,7 @@ public void JsonOutputter_DatatypeFloat_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 3.5F, 0F }; + object[] values = { 3.5F, 0F }; var row = new USqlRow(schema, values); var expected = "{\"a\":3.5,\"b\":0.0}"; @@ -140,7 +140,7 @@ public void JsonOutputter_DatatypeNullableFloat_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 3.5F, 0F, null }; + object[] values = { 3.5F, 0F, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":3.5,\"b\":0.0}"; @@ -156,7 +156,7 @@ public void JsonOutputter_DatatypeDouble_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 3.5D, 0D }; + object[] values = { 3.5D, 0D }; var row = new USqlRow(schema, values); var expected = "{\"a\":3.5,\"b\":0.0}"; @@ -173,7 +173,7 @@ public void JsonOutputter_DatatypeNullableDouble_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 3.5, 0D, null }; + object[] values = { 3.5, 0D, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":3.5,\"b\":0.0}"; @@ -189,7 +189,7 @@ public void JsonOutputter_DatatypeDecimal_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 350.5M, 0M }; + object[] values = { 350.5M, 0M }; var row = new USqlRow(schema, values); var expected = "{\"a\":350.5,\"b\":0.0}"; @@ -206,7 +206,7 @@ public void JsonOutputter_DatatypeNullableDecimal_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 350.5M, 0M, null }; + object[] values = { 350.5M, 0M, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":350.5,\"b\":0.0}"; @@ -226,7 +226,7 @@ public void JsonOutputter_DatatypeByte_Outputted() byte a = 2; byte b = 4; - object[] values = new object[2] { a, b }; + object[] values = { a, b }; var row = new USqlRow(schema, values); var expected = "{\"a\":2,\"b\":4}"; @@ -246,7 +246,7 @@ public void JsonOutputter_DatatypeNullableBytes_Outputted() byte? a = 2; byte? b = null; - object[] values = new object[2] { a, b }; + object[] values = { a, b }; var row = new USqlRow(schema, values); var expected = "{\"a\":2}"; @@ -262,7 +262,7 @@ public void JsonOutputter_DatatypeBoolean_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { true, false }; + object[] values = { true, false }; var row = new USqlRow(schema, values); var expected = "{\"a\":true,\"b\":false}"; @@ -279,7 +279,7 @@ public void JsonOutputter_DatatypeNullableBoolean_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { true, false, null }; + object[] values = { true, false, null }; var row = new USqlRow(schema, values); var expected = "{\"a\":true,\"b\":false}"; @@ -296,7 +296,7 @@ public void JsonOutputter_DatatypeString_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { "test", "", null }; + object[] values = { "test", "", null }; var row = new USqlRow(schema, values); var expected = "{\"a\":\"test\",\"b\":\"\"}"; @@ -312,7 +312,7 @@ public void JsonOutputter_DatatypeChar_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { 'a', ' ' }; + object[] values = { 'a', ' ' }; var row = new USqlRow(schema, values); var expected = "{\"a\":\"a\",\"b\":\" \"}"; @@ -329,7 +329,7 @@ public void JsonOutputter_DatatypeNullableChar_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = new object[3] { 'a', ' ', null }; + object[] values = { 'a', ' ', null }; var row = new USqlRow(schema, values); var expected = "{\"a\":\"a\",\"b\":\" \"}"; @@ -344,7 +344,7 @@ public void JsonOutputter_DatatypeDateTime_Outputted() USqlSchema schema = new USqlSchema( new USqlColumn("a") ); - object[] values = new object[1] { new DateTime(2010, 01, 05) }; + object[] values = { new DateTime(2010, 01, 05) }; var row = new USqlRow(schema, values); var expected = "{\"a\":\"2010-01-05T00:00:00\"}"; @@ -360,7 +360,7 @@ public void JsonOutputter_DatatypeNullableDateTime_Outputted() new USqlColumn("a"), new USqlColumn("b") ); - object[] values = new object[2] { new DateTime(2010, 01, 05), null }; + object[] values = { new DateTime(2010, 01, 05), null }; var row = new USqlRow(schema, values); var expected = "{\"a\":\"2010-01-05T00:00:00\"}"; @@ -385,18 +385,18 @@ public void JsonOutputter_DatatypeArrayOfPrimitiveTypes_Outputted() new USqlColumn("j"), new USqlColumn("k") ); - object[] values = new object[11] { - new short?[3] { 0, 1, null }, - new int?[3] { 0, 1, null }, - new long?[3] { 9223372036854775807, -9223372036854775807, null }, - new float?[3] { 3.5F, 0F, null }, - new double?[3] { 3.5D, 0D, null }, - new decimal?[3] { 205.2M, 0M, null }, - new byte?[3] { 2, 4, null }, - new bool?[3] { true, false, null }, - new string[3] { "test", "", null }, - new char?[3] { 'a', ' ', null }, - new DateTime?[3] { new DateTime(2010, 01, 05), new DateTime(2015, 05, 06), null } + object[] values = { + new short?[] { 0, 1, null }, + new int?[] { 0, 1, null }, + new long?[] { 9223372036854775807, -9223372036854775807, null }, + new float?[] { 3.5F, 0F, null }, + new double?[] { 3.5D, 0D, null }, + new decimal?[] { 205.2M, 0M, null }, + new byte?[] { 2, 4, null }, + new bool?[] { true, false, null }, + new[] { "test", "", null }, + new char?[] { 'a', ' ', null }, + new DateTime?[] { new DateTime(2010, 01, 05), new DateTime(2015, 05, 06), null } }; var row = new USqlRow(schema, values); @@ -440,7 +440,7 @@ public void JsonOutputter_DatatypeDictionaryOfPrimitiveTypes_Outputted() { "DateTime", new DateTime(2015, 05, 06) } }; - object[] values = new object[1] { dict }; + object[] values = { dict }; var row = new USqlRow(schema, values); @@ -483,7 +483,7 @@ public void JsonOutputter_DatatypeSqlMapOfPrimitiveTypes_Outputted() { "DateTime", new DateTime(2015, 05, 06) } }); - object[] values = new object[1] { map }; + object[] values = { map }; var row = new USqlRow(schema, values); @@ -513,7 +513,7 @@ public void JsonOutputter_DatatypeKeyValuePairPrimitiveTypes_Outputted() ); var keyValuePair = new KeyValuePair("int", 3); - object[] values = new object[1] { keyValuePair }; + object[] values = { keyValuePair }; var row = new USqlRow(schema, values); @@ -530,8 +530,7 @@ public void JsonOutputter_DatatypeArrayOfMaps_Outputted() new USqlColumn[]>("a") ); - Dictionary[] dictArray = new Dictionary[2] - { + Dictionary[] dictArray = { new Dictionary { { "test1", "asd" }, @@ -544,7 +543,7 @@ public void JsonOutputter_DatatypeArrayOfMaps_Outputted() } }; - object[] values = new object[1] { dictArray }; + object[] values = { dictArray }; var row = new USqlRow(schema, values); @@ -563,11 +562,11 @@ public void JsonOutputter_DatatypeMapOfArrays_Outputted() Dictionary dict = new Dictionary { - { "test1", new int[]{ 2, 3 } }, - { "test2", new string[]{ "asd", "" } }, + { "test1", new[]{ 2, 3 } }, + { "test2", new[]{ "asd", "" } }, }; - object[] values = new object[1] { dict }; + object[] values = { dict }; var row = new USqlRow(schema, values); @@ -587,16 +586,16 @@ public void JsonOutputter_DatatypeMapOfArrays_MultiRow_Outputted() List rows = new List(); Dictionary dict = new Dictionary { - { "test1", new int[]{ 2, 3 } }, - { "test2", new string[]{ "asd", "" } }, + { "test1", new[]{ 2, 3 } }, + { "test2", new[]{ "asd", "" } }, }; rows.Add(new USqlRow(schema, new object[] { dict })); dict = new Dictionary { - { "test3", new int[]{ 1, 4 } }, - { "test4", new string[]{ "foo", "bar" } }, + { "test3", new[]{ 1, 4 } }, + { "test4", new[]{ "foo", "bar" } }, }; rows.Add(new USqlRow(schema, new object[] { dict })); @@ -620,10 +619,10 @@ public void JsonOutputter_DatatypeComplex_Outputted() Dictionary complex = new Dictionary { { - "test1", new Dictionary(){ { "nested", 1}, { "nestedArray", new int[] { 1, 2 } } } + "test1", new Dictionary(){ { "nested", 1}, { "nestedArray", new[] { 1, 2 } } } }, { - "test2", new Dictionary[] + "test2", new[] { new Dictionary { @@ -639,7 +638,7 @@ public void JsonOutputter_DatatypeComplex_Outputted() } }; - object[] values = new object[1] { complex }; + object[] values = { complex }; var row = new USqlRow(schema, values); From 36c868e40d54cfd02e729d3e20d92c7e37a42eb1 Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 17 Dec 2018 14:22:31 +0100 Subject: [PATCH 4/5] code cleanup --- .../JsonLinesOutputterTests.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs index 8723df3..5a4a795 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs @@ -1,12 +1,12 @@ using System; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Microsoft.Analytics.Samples.Formats.Json; -using System.IO; -using Microsoft.Analytics.UnitTest; -using Microsoft.Analytics.Interfaces; using System.Collections.Generic; +using System.IO; using System.Text; +using Microsoft.Analytics.Interfaces; +using Microsoft.Analytics.Samples.Formats.Json; using Microsoft.Analytics.Types.Sql; +using Microsoft.Analytics.UnitTest; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.Analytics.Samples.Formats.Tests { @@ -534,12 +534,12 @@ public void JsonOutputter_DatatypeArrayOfMaps_Outputted() new Dictionary { { "test1", "asd" }, - { "test2", 3 }, + { "test2", 3 } }, new Dictionary { { "test1", "das" }, - { "test2", 3 }, + { "test2", 3 } } }; @@ -563,7 +563,7 @@ public void JsonOutputter_DatatypeMapOfArrays_Outputted() Dictionary dict = new Dictionary { { "test1", new[]{ 2, 3 } }, - { "test2", new[]{ "asd", "" } }, + { "test2", new[]{ "asd", "" } } }; object[] values = { dict }; @@ -587,7 +587,7 @@ public void JsonOutputter_DatatypeMapOfArrays_MultiRow_Outputted() Dictionary dict = new Dictionary { { "test1", new[]{ 2, 3 } }, - { "test2", new[]{ "asd", "" } }, + { "test2", new[]{ "asd", "" } } }; rows.Add(new USqlRow(schema, new object[] { dict })); @@ -595,7 +595,7 @@ public void JsonOutputter_DatatypeMapOfArrays_MultiRow_Outputted() dict = new Dictionary { { "test3", new[]{ 1, 4 } }, - { "test4", new[]{ "foo", "bar" } }, + { "test4", new[]{ "foo", "bar" } } }; rows.Add(new USqlRow(schema, new object[] { dict })); @@ -619,7 +619,7 @@ public void JsonOutputter_DatatypeComplex_Outputted() Dictionary complex = new Dictionary { { - "test1", new Dictionary(){ { "nested", 1}, { "nestedArray", new[] { 1, 2 } } } + "test1", new Dictionary { { "nested", 1}, { "nestedArray", new[] { 1, 2 } } } }, { "test2", new[] @@ -627,12 +627,12 @@ public void JsonOutputter_DatatypeComplex_Outputted() new Dictionary { { "test1", "asd" }, - { "test2", 3 }, + { "test2", 3 } }, new Dictionary { { "test1", "das" }, - { "test2", 3 }, + { "test2", 3 } } } } From 1fca7b6e992ac579098e46a7b636f7c4fa6f231d Mon Sep 17 00:00:00 2001 From: sg Date: Mon, 17 Dec 2018 16:24:10 +0100 Subject: [PATCH 5/5] added newline tests --- .../JsonLinesOutputterTests.cs | 8 ++++---- .../Json/JsonLinesOutputter.cs | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs index 5a4a795..f04682b 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats.Tests/JsonLinesOutputterTests.cs @@ -296,10 +296,10 @@ public void JsonOutputter_DatatypeString_Outputted() new USqlColumn("b"), new USqlColumn("c") ); - object[] values = { "test", "", null }; + object[] values = { "test", "foo\r\nbar", null }; var row = new USqlRow(schema, values); - var expected = "{\"a\":\"test\",\"b\":\"\"}"; + var expected = "{\"a\":\"test\",\"b\":\"foo\\r\\nbar\"}"; var actual = GetOutputterResult(row); Assert.AreEqual(expected, actual); @@ -601,7 +601,7 @@ public void JsonOutputter_DatatypeMapOfArrays_MultiRow_Outputted() rows.Add(new USqlRow(schema, new object[] { dict })); - var expected = "{\"a\":{\"test1\":[2,3],\"test2\":[\"asd\",\"\"]}}"+ + var expected = "{\"a\":{\"test1\":[2,3],\"test2\":[\"asd\",\"\"]}}" + Environment.NewLine + "{\"a\":{\"test3\":[1,4],\"test4\":[\"foo\",\"bar\"]}}"; var actual = GetOutputterResult(rows); @@ -671,7 +671,7 @@ private string GetOutputterResult(IEnumerable rows) foreach (IRow row in rows) { outputter.Output(row, unstructuredWriter); - } + } outputter.Close(); var output = Encoding.ASCII.GetString(ms.ToArray()).TrimEnd(); return output; diff --git a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs index e9893cd..07acec4 100644 --- a/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs +++ b/Examples/DataFormats/Microsoft.Analytics.Samples.Formats/Json/JsonLinesOutputter.cs @@ -35,6 +35,7 @@ namespace Microsoft.Analytics.Samples.Formats.Json /// ... /// Notice that this outputter doesn't require atomic output, /// since it produces standalone JSON documents as opposed to a single JSON array. + /// Adds an extra newline at the end of the file. /// [SqlUserDefinedOutputter(AtomicFileProcessing = false)] public class JsonLinesOutputter : IOutputter