From c753f6b476bf1ca4325497aba7b0f33667ce937b Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 16 Jul 2025 15:04:45 +0300 Subject: [PATCH 01/11] text json --- mix.exs | 2 +- mix.lock | 2 +- test/ecto/integration/json_test.exs | 33 ++++++++++++++++++++++++++++- test/support/migrations.ex | 7 +++--- test/test_helper.exs | 19 +++++++++++++++-- 5 files changed, 54 insertions(+), 9 deletions(-) diff --git a/mix.exs b/mix.exs index dbfa92fb..fe8fa8e0 100644 --- a/mix.exs +++ b/mix.exs @@ -39,7 +39,7 @@ defmodule EctoCh.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ch, "~> 0.4.0"}, + {:ch, github: "plausible/ch", branch: "text-json"}, {:ecto_sql, "~> 3.13.0"}, {:benchee, "~> 1.1", only: :bench}, {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false}, diff --git a/mix.lock b/mix.lock index f5356d2e..57166b44 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "benchee": {:hex, :benchee, "1.4.0", "9f1f96a30ac80bab94faad644b39a9031d5632e517416a8ab0a6b0ac4df124ce", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "299cd10dd8ce51c9ea3ddb74bb150f93d25e968f93e4c1fa31698a8e4fa5d715"}, - "ch": {:hex, :ch, "0.4.1", "716fc326a0d29212a35c15e5350355550ff1290a244e6f37bf3eac4318aeeb76", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "e6a4cf90d22030afde77e1a2895ebba2888032ccc5f1bec947b26b90a9152ffc"}, + "ch": {:git, "https://github.com/plausible/ch.git", "2e5fc8cc1a5af86852c17ad5f42c84fc2a5eced9", [branch: "text-json"]}, "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 478e5cc9..741b581f 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -1,10 +1,12 @@ defmodule Ecto.Integration.JsonTest do use Ecto.Integration.Case + @moduletag :json + alias Ecto.Integration.TestRepo alias EctoClickHouse.Integration.Setting - @tag skip: true + @tag :skip test "serializes json correctly" do # Insert a record purposefully with atoms as the map key. We are going to # verify later they were coerced into strings. @@ -23,4 +25,33 @@ defmodule Ecto.Integration.JsonTest do [setting.id] ) end + + defmodule SemiStructured do + use Ecto.Schema + + @primary_key false + schema "semi_structured" do + field :json, Ch, type: "JSON" + field :time, :naive_datetime + end + end + + test "it works" do + TestRepo.query!(""" + CREATE TABLE semi_structured ( + json JSON, + time DateTime + ) ENGINE MergeTree ORDER BY time + """) + + TestRepo.insert!(%SemiStructured{json: %{foo: "bar", baz: 42}, time: ~N[2023-10-01 12:00:00]}) + + assert [ + %SemiStructured{ + json: %{"foo" => "bar", "baz" => "42"}, + time: ~N[2023-10-01 12:00:00] + } + ] = + TestRepo.all(SemiStructured) + end end diff --git a/test/support/migrations.ex b/test/support/migrations.ex index 8de3a861..0a837d66 100644 --- a/test/support/migrations.ex +++ b/test/support/migrations.ex @@ -189,9 +189,8 @@ defmodule EctoClickHouse.Integration.Migration do add :z, :Float64 end - # TODO - # create table(:settings, primary_key: false, engine: "MergeTree") do - # add :properties, :JSON - # end + create table(:settings, primary_key: false, engine: "MergeTree", options: "order by tuple()") do + add :properties, :JSON + end end end diff --git a/test/test_helper.exs b/test/test_helper.exs index 594202dd..7b28eb18 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -28,7 +28,12 @@ alias Ecto.Integration.TestRepo Application.put_env(:ecto_ch, TestRepo, adapter: Ecto.Adapters.ClickHouse, database: "ecto_ch_test", - show_sensitive_data_on_connection_error: true + show_sensitive_data_on_connection_error: true, + settings: [ + enable_json_type: 1, + output_format_binary_write_json_as_string: 1, + input_format_binary_read_json_as_string: 1 + ] ) {:ok, _} = Ecto.Adapters.ClickHouse.ensure_all_started(TestRepo.config(), :temporary) @@ -39,4 +44,14 @@ _ = Ecto.Adapters.ClickHouse.storage_down(TestRepo.config()) {:ok, _} = TestRepo.start_link() :ok = Ecto.Migrator.up(TestRepo, 0, EctoClickHouse.Integration.Migration, log: false) -ExUnit.start() +%{rows: [[ch_version]]} = TestRepo.query!("select version()") + +exclude = + if ch_version >= "25" do + [] + else + # JSON type is not supported in ClickHouse < 25 + [:json] + end + +ExUnit.start(exclude: exclude) From ae960c57fa380f5cccd00c95624e1678cfc9605e Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 16 Jul 2025 15:13:07 +0300 Subject: [PATCH 02/11] support casting --- mix.lock | 2 +- test/ecto/integration/json_test.exs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 57166b44..51924693 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "benchee": {:hex, :benchee, "1.4.0", "9f1f96a30ac80bab94faad644b39a9031d5632e517416a8ab0a6b0ac4df124ce", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "299cd10dd8ce51c9ea3ddb74bb150f93d25e968f93e4c1fa31698a8e4fa5d715"}, - "ch": {:git, "https://github.com/plausible/ch.git", "2e5fc8cc1a5af86852c17ad5f42c84fc2a5eced9", [branch: "text-json"]}, + "ch": {:git, "https://github.com/plausible/ch.git", "abeb446f03d3624f23626822469b7ef8bf42e3fa", [branch: "text-json"]}, "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 741b581f..60fbff69 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -44,7 +44,15 @@ defmodule Ecto.Integration.JsonTest do ) ENGINE MergeTree ORDER BY time """) - TestRepo.insert!(%SemiStructured{json: %{foo: "bar", baz: 42}, time: ~N[2023-10-01 12:00:00]}) + %SemiStructured{} + |> Ecto.Changeset.cast( + %{ + json: %{"foo" => "bar", "baz" => 42}, + time: ~N[2023-10-01 12:00:00] + }, + [:json, :time] + ) + |> TestRepo.insert!() assert [ %SemiStructured{ From acb54958370d317612e09bb58371c1264ec3a5b2 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 16 Jul 2025 20:06:57 +0300 Subject: [PATCH 03/11] add insert_all to the test --- test/ecto/integration/json_test.exs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 60fbff69..3d42902b 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -1,5 +1,6 @@ defmodule Ecto.Integration.JsonTest do use Ecto.Integration.Case + import Ecto.Query, only: [from: 2] @moduletag :json @@ -36,7 +37,7 @@ defmodule Ecto.Integration.JsonTest do end end - test "it works" do + test "basic" do TestRepo.query!(""" CREATE TABLE semi_structured ( json JSON, @@ -47,19 +48,22 @@ defmodule Ecto.Integration.JsonTest do %SemiStructured{} |> Ecto.Changeset.cast( %{ - json: %{"foo" => "bar", "baz" => 42}, + json: %{"from" => "insert"}, time: ~N[2023-10-01 12:00:00] }, [:json, :time] ) |> TestRepo.insert!() - assert [ - %SemiStructured{ - json: %{"foo" => "bar", "baz" => "42"}, - time: ~N[2023-10-01 12:00:00] - } - ] = - TestRepo.all(SemiStructured) + TestRepo.insert_all(SemiStructured, [ + %{json: %{"from" => "insert_all"}, time: ~N[2023-10-01 13:00:00]}, + %{json: %{"from" => "another_insert_all"}, time: ~N[2023-10-01 13:01:00]} + ]) + + assert TestRepo.all(from s in SemiStructured, select: s.json, order_by: s.time) == [ + %{"from" => "insert"}, + %{"from" => "insert_all"}, + %{"from" => "another_insert_all"} + ] end end From 38eb62f0e72c13a533997b7a8bf1fafcfa5c435f Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 16 Jul 2025 20:14:28 +0300 Subject: [PATCH 04/11] add more involved test --- test/ecto/integration/json_test.exs | 216 ++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 3d42902b..c494a036 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -45,6 +45,8 @@ defmodule Ecto.Integration.JsonTest do ) ENGINE MergeTree ORDER BY time """) + on_exit(fn -> TestRepo.query!("DROP TABLE semi_structured") end) + %SemiStructured{} |> Ecto.Changeset.cast( %{ @@ -66,4 +68,218 @@ defmodule Ecto.Integration.JsonTest do %{"from" => "another_insert_all"} ] end + + # https://github.com/plausible/ecto_ch/pull/233#issuecomment-3079317842 + + defmodule TokenInfoSchema do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "token_infos" do + field :mint, Ch, type: "String" + field :data, Ch, type: "JSON", source: :data + field :created_at, Ch, type: "DateTime" + end + end + + test "token_info_schema" do + TestRepo.query!(""" + create table token_infos( + mint String, + data JSON, + created_at DateTime + ) engine = MergeTree order by created_at + """) + + on_exit(fn -> TestRepo.query!("DROP TABLE token_infos") end) + + TestRepo.insert_all(TokenInfoSchema, [ + %{ + data: %{ + "authorities" => [ + %{"address" => "2wmVCSfPxGPjrnMMn7rchp4uaeoTqN39mXFC2zhPdri9", "scopes" => ["full"]} + ], + "burnt" => false, + "compression" => %{ + "asset_hash" => "", + "compressed" => false, + "creator_hash" => "", + "data_hash" => "", + "eligible" => false, + "leaf_id" => 0, + "seq" => 0, + "tree" => "" + }, + "content" => %{ + "$schema" => "https://schema.metaplex.com/nft1.0.json", + "files" => [ + %{ + "cdn_uri" => + "https://cdn.helius-rpc.com/cdn-cgi/image//https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png", + "mime" => "image/png", + "uri" => + "https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png" + } + ], + "json_uri" => "", + "links" => %{ + "image" => + "https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png" + }, + "metadata" => %{"name" => "USD Coin", "symbol" => "USDC"} + }, + "creators" => [], + "grouping" => [], + "id" => "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + "interface" => "FungibleToken", + "mutable" => true, + "ownership" => %{ + "delegate" => nil, + "delegated" => false, + "frozen" => false, + "owner" => "", + "ownership_model" => "token" + }, + "royalty" => %{ + "basis_points" => 0, + "locked" => false, + "percent" => 0.0, + "primary_sale_happened" => false, + "royalty_model" => "creators", + "target" => nil + }, + "supply" => nil, + "token_info" => %{ + "decimals" => 6, + "freeze_authority" => "7dGbd2QZcCKcTndnHcTL8q7SMVXAkp688NTQYwrRCrar", + "mint_authority" => "BJE5MMbqXjVwjAF7oxwPYXnTXDyspzZyt4vwenNw5ruG", + "price_info" => %{"currency" => "USDC", "price_per_token" => 0.999867}, + "supply" => 8_276_375_974_708_499, + "symbol" => "USDC", + "token_program" => "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + } + }, + mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", + created_at: ~N[2025-07-16 16:04:47] + }, + %{ + data: %{ + "authorities" => [ + %{"address" => "2RtGg6fsFiiF1EQzHqbd66AhW7R5bWeQGpTbv2UMkCdW", "scopes" => ["full"]} + ], + "burnt" => false, + "compression" => %{ + "asset_hash" => "", + "compressed" => false, + "creator_hash" => "", + "data_hash" => "", + "eligible" => false, + "leaf_id" => 0, + "seq" => 0, + "tree" => "" + }, + "content" => %{ + "$schema" => "https://schema.metaplex.com/nft1.0.json", + "files" => [ + %{ + "cdn_uri" => + "https://cdn.helius-rpc.com/cdn-cgi/image//https://madlads.s3.us-west-2.amazonaws.com/images/8420.png", + "mime" => "image/png", + "uri" => "https://madlads.s3.us-west-2.amazonaws.com/images/8420.png" + }, + %{ + "cdn_uri" => + "https://cdn.helius-rpc.com/cdn-cgi/image//https://arweave.net/qJ5B6fx5hEt4P7XbicbJQRyTcbyLaV-OQNA1KjzdqOQ/0.png", + "mime" => "image/png", + "uri" => "https://arweave.net/qJ5B6fx5hEt4P7XbicbJQRyTcbyLaV-OQNA1KjzdqOQ/0.png" + } + ], + "json_uri" => "https://madlads.s3.us-west-2.amazonaws.com/json/8420.json", + "links" => %{ + "external_url" => "https://madlads.com", + "image" => "https://madlads.s3.us-west-2.amazonaws.com/images/8420.png" + }, + "metadata" => %{ + "attributes" => [ + %{"trait_type" => "Gender", "value" => "Male"}, + %{"trait_type" => "Type", "value" => "King"}, + %{"trait_type" => "Expression", "value" => "Royal"}, + %{"trait_type" => "Hat", "value" => "Mad Crown"}, + %{"trait_type" => "Eyes", "value" => "Madness"}, + %{"trait_type" => "Clothing", "value" => "Mad Armor"}, + %{"trait_type" => "Background", "value" => "Royal Rug"} + ], + "description" => "Fock it.", + "name" => "Mad Lads #8420", + "symbol" => "MAD", + "token_standard" => "ProgrammableNonFungible" + } + }, + "creators" => [ + %{ + "address" => "5XvhfmRjwXkGp3jHGmaKpqeerNYjkuZZBYLVQYdeVcRv", + "share" => 0, + "verified" => true + }, + %{ + "address" => "2RtGg6fsFiiF1EQzHqbd66AhW7R5bWeQGpTbv2UMkCdW", + "share" => 100, + "verified" => true + } + ], + "grouping" => [ + %{ + "group_key" => "collection", + "group_value" => "J1S9H3QjnRtBbbuD4HjPV6RpRhwuk4zKbxsnCHuTgh9w" + } + ], + "id" => "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", + "interface" => "ProgrammableNFT", + "mutable" => true, + "ownership" => %{ + "delegate" => nil, + "delegated" => false, + "frozen" => true, + "owner" => "D3ftM66SZMdbCHiV9wBAFxoqqA8ex76nJnmVLbGy6vwp", + "ownership_model" => "single" + }, + "royalty" => %{ + "basis_points" => 420, + "locked" => false, + "percent" => 0.042, + "primary_sale_happened" => true, + "royalty_model" => "creators", + "target" => nil + }, + "supply" => %{ + "edition_nonce" => 254, + "print_current_supply" => 0, + "print_max_supply" => 0 + }, + "token_info" => %{ + "decimals" => 0, + "freeze_authority" => "TdMA45ZnakQCBt5XUvm7ib2htKuTWdcgGKu1eUGrDyJ", + "mint_authority" => "TdMA45ZnakQCBt5XUvm7ib2htKuTWdcgGKu1eUGrDyJ", + "supply" => 1, + "token_program" => "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" + } + }, + mint: "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", + created_at: ~N[2025-07-16 16:04:47] + } + ]) + + assert TestRepo.all( + from t in TokenInfoSchema, + order_by: t.created_at, + select: %{ + mint: t.mint, + basis_points: fragment("?.royalty.basis_points::String", t.data) + } + ) == [ + %{mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", basis_points: "0"}, + %{mint: "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", basis_points: "420"} + ] + end end From 05e5dbcabcdbd0cd9eadc1a8827b77359916605d Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 00:39:06 +0300 Subject: [PATCH 05/11] try again --- test/ecto/integration/json_test.exs | 187 ++-------------------------- 1 file changed, 13 insertions(+), 174 deletions(-) diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index c494a036..7556c919 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -94,192 +94,31 @@ defmodule Ecto.Integration.JsonTest do on_exit(fn -> TestRepo.query!("DROP TABLE token_infos") end) - TestRepo.insert_all(TokenInfoSchema, [ + missing_tokens = [ %{ - data: %{ - "authorities" => [ - %{"address" => "2wmVCSfPxGPjrnMMn7rchp4uaeoTqN39mXFC2zhPdri9", "scopes" => ["full"]} - ], - "burnt" => false, - "compression" => %{ - "asset_hash" => "", - "compressed" => false, - "creator_hash" => "", - "data_hash" => "", - "eligible" => false, - "leaf_id" => 0, - "seq" => 0, - "tree" => "" - }, - "content" => %{ - "$schema" => "https://schema.metaplex.com/nft1.0.json", - "files" => [ - %{ - "cdn_uri" => - "https://cdn.helius-rpc.com/cdn-cgi/image//https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png", - "mime" => "image/png", - "uri" => - "https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png" - } - ], - "json_uri" => "", - "links" => %{ - "image" => - "https://raw.githubusercontent.com/solana-labs/token-list/main/assets/mainnet/EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v/logo.png" - }, - "metadata" => %{"name" => "USD Coin", "symbol" => "USDC"} - }, - "creators" => [], - "grouping" => [], - "id" => "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", - "interface" => "FungibleToken", - "mutable" => true, - "ownership" => %{ - "delegate" => nil, - "delegated" => false, - "frozen" => false, - "owner" => "", - "ownership_model" => "token" - }, - "royalty" => %{ - "basis_points" => 0, - "locked" => false, - "percent" => 0.0, - "primary_sale_happened" => false, - "royalty_model" => "creators", - "target" => nil - }, - "supply" => nil, - "token_info" => %{ - "decimals" => 6, - "freeze_authority" => "7dGbd2QZcCKcTndnHcTL8q7SMVXAkp688NTQYwrRCrar", - "mint_authority" => "BJE5MMbqXjVwjAF7oxwPYXnTXDyspzZyt4vwenNw5ruG", - "price_info" => %{"currency" => "USDC", "price_per_token" => 0.999867}, - "supply" => 8_276_375_974_708_499, - "symbol" => "USDC", - "token_program" => "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" - } - }, - mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", - created_at: ~N[2025-07-16 16:04:47] + mint: "123", + data: %{"name" => "Test", "nested" => %{"name" => "Test", "arr" => ["abc", "b=deb"]}}, + created_at: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) }, %{ - data: %{ - "authorities" => [ - %{"address" => "2RtGg6fsFiiF1EQzHqbd66AhW7R5bWeQGpTbv2UMkCdW", "scopes" => ["full"]} - ], - "burnt" => false, - "compression" => %{ - "asset_hash" => "", - "compressed" => false, - "creator_hash" => "", - "data_hash" => "", - "eligible" => false, - "leaf_id" => 0, - "seq" => 0, - "tree" => "" - }, - "content" => %{ - "$schema" => "https://schema.metaplex.com/nft1.0.json", - "files" => [ - %{ - "cdn_uri" => - "https://cdn.helius-rpc.com/cdn-cgi/image//https://madlads.s3.us-west-2.amazonaws.com/images/8420.png", - "mime" => "image/png", - "uri" => "https://madlads.s3.us-west-2.amazonaws.com/images/8420.png" - }, - %{ - "cdn_uri" => - "https://cdn.helius-rpc.com/cdn-cgi/image//https://arweave.net/qJ5B6fx5hEt4P7XbicbJQRyTcbyLaV-OQNA1KjzdqOQ/0.png", - "mime" => "image/png", - "uri" => "https://arweave.net/qJ5B6fx5hEt4P7XbicbJQRyTcbyLaV-OQNA1KjzdqOQ/0.png" - } - ], - "json_uri" => "https://madlads.s3.us-west-2.amazonaws.com/json/8420.json", - "links" => %{ - "external_url" => "https://madlads.com", - "image" => "https://madlads.s3.us-west-2.amazonaws.com/images/8420.png" - }, - "metadata" => %{ - "attributes" => [ - %{"trait_type" => "Gender", "value" => "Male"}, - %{"trait_type" => "Type", "value" => "King"}, - %{"trait_type" => "Expression", "value" => "Royal"}, - %{"trait_type" => "Hat", "value" => "Mad Crown"}, - %{"trait_type" => "Eyes", "value" => "Madness"}, - %{"trait_type" => "Clothing", "value" => "Mad Armor"}, - %{"trait_type" => "Background", "value" => "Royal Rug"} - ], - "description" => "Fock it.", - "name" => "Mad Lads #8420", - "symbol" => "MAD", - "token_standard" => "ProgrammableNonFungible" - } - }, - "creators" => [ - %{ - "address" => "5XvhfmRjwXkGp3jHGmaKpqeerNYjkuZZBYLVQYdeVcRv", - "share" => 0, - "verified" => true - }, - %{ - "address" => "2RtGg6fsFiiF1EQzHqbd66AhW7R5bWeQGpTbv2UMkCdW", - "share" => 100, - "verified" => true - } - ], - "grouping" => [ - %{ - "group_key" => "collection", - "group_value" => "J1S9H3QjnRtBbbuD4HjPV6RpRhwuk4zKbxsnCHuTgh9w" - } - ], - "id" => "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", - "interface" => "ProgrammableNFT", - "mutable" => true, - "ownership" => %{ - "delegate" => nil, - "delegated" => false, - "frozen" => true, - "owner" => "D3ftM66SZMdbCHiV9wBAFxoqqA8ex76nJnmVLbGy6vwp", - "ownership_model" => "single" - }, - "royalty" => %{ - "basis_points" => 420, - "locked" => false, - "percent" => 0.042, - "primary_sale_happened" => true, - "royalty_model" => "creators", - "target" => nil - }, - "supply" => %{ - "edition_nonce" => 254, - "print_current_supply" => 0, - "print_max_supply" => 0 - }, - "token_info" => %{ - "decimals" => 0, - "freeze_authority" => "TdMA45ZnakQCBt5XUvm7ib2htKuTWdcgGKu1eUGrDyJ", - "mint_authority" => "TdMA45ZnakQCBt5XUvm7ib2htKuTWdcgGKu1eUGrDyJ", - "supply" => 1, - "token_program" => "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" - } - }, - mint: "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", - created_at: ~N[2025-07-16 16:04:47] + mint: "325", + data: %{"name" => "Test", "nested" => %{"name" => "Test", "arr" => ["abc", "b=deb"]}}, + created_at: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) } - ]) + ] + + assert {2, nil} = TestRepo.insert_all(TokenInfoSchema, missing_tokens) assert TestRepo.all( from t in TokenInfoSchema, order_by: t.created_at, select: %{ mint: t.mint, - basis_points: fragment("?.royalty.basis_points::String", t.data) + name: fragment("?.nested.name::text", t.data) } ) == [ - %{mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", basis_points: "0"}, - %{mint: "F9Lw3ki3hJ7PF9HQXsBzoY8GyE6sPoEZZdXJBsTTD2rk", basis_points: "420"} + %{mint: "123", name: "Test"}, + %{mint: "325", name: "Test"} ] end end From 60ed8bf49aee753d2f43ff15b53d39361796cd7e Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:24:25 +0300 Subject: [PATCH 06/11] use released ch --- CHANGELOG.md | 6 ++++++ mix.exs | 2 +- test/test_helper.exs | 7 +------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a062c8d9..33b205bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- add [Time](https://clickhouse.com/docs/sql-reference/data-types/time) and [Time64](https://clickhouse.com/docs/sql-reference/data-types/time64) types support +- add [Variant](https://clickhouse.com/docs/sql-reference/data-types/variant) type support +- add [JSON](https://clickhouse.com/docs/sql-reference/data-types/newjson) type support + ## 0.7.1 (2025-07-07) - update [Ch](https://github.com/plausible/ch) (our ClickHouse client) to v0.4.x diff --git a/mix.exs b/mix.exs index fe8fa8e0..7499574c 100644 --- a/mix.exs +++ b/mix.exs @@ -39,7 +39,7 @@ defmodule EctoCh.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ch, github: "plausible/ch", branch: "text-json"}, + {:ch, "~> 0.5.0"}, {:ecto_sql, "~> 3.13.0"}, {:benchee, "~> 1.1", only: :bench}, {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false}, diff --git a/test/test_helper.exs b/test/test_helper.exs index 7b28eb18..a536e79f 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -28,12 +28,7 @@ alias Ecto.Integration.TestRepo Application.put_env(:ecto_ch, TestRepo, adapter: Ecto.Adapters.ClickHouse, database: "ecto_ch_test", - show_sensitive_data_on_connection_error: true, - settings: [ - enable_json_type: 1, - output_format_binary_write_json_as_string: 1, - input_format_binary_read_json_as_string: 1 - ] + show_sensitive_data_on_connection_error: true ) {:ok, _} = Ecto.Adapters.ClickHouse.ensure_all_started(TestRepo.config(), :temporary) From 8b625b04bef3b939fa5d1cad0851aa9b045f502a Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:25:49 +0300 Subject: [PATCH 07/11] deps.get --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 51924693..f0bf9d19 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "benchee": {:hex, :benchee, "1.4.0", "9f1f96a30ac80bab94faad644b39a9031d5632e517416a8ab0a6b0ac4df124ce", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "299cd10dd8ce51c9ea3ddb74bb150f93d25e968f93e4c1fa31698a8e4fa5d715"}, - "ch": {:git, "https://github.com/plausible/ch.git", "abeb446f03d3624f23626822469b7ef8bf42e3fa", [branch: "text-json"]}, + "ch": {:hex, :ch, "0.5.0", "e1047b9a650d34ff5b001a149aa88ebfdf3d31a481e3e255747cf1635e9d54eb", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "119100210a128fc4d5ef4e7531a1256f4ee686afc85241392e93bb1da0b56967"}, "db_connection": {:hex, :db_connection, "2.8.0", "64fd82cfa6d8e25ec6660cea73e92a4cbc6a18b31343910427b702838c4b33b2", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "008399dae5eee1bf5caa6e86d204dcb44242c82b1ed5e22c881f2c34da201b15"}, "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, From ac4d42d78e1c042df6bbb984c6250cdf44758f40 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:26:54 +0300 Subject: [PATCH 08/11] eh --- test/test_helper.exs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index a536e79f..594202dd 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -39,14 +39,4 @@ _ = Ecto.Adapters.ClickHouse.storage_down(TestRepo.config()) {:ok, _} = TestRepo.start_link() :ok = Ecto.Migrator.up(TestRepo, 0, EctoClickHouse.Integration.Migration, log: false) -%{rows: [[ch_version]]} = TestRepo.query!("select version()") - -exclude = - if ch_version >= "25" do - [] - else - # JSON type is not supported in ClickHouse < 25 - [:json] - end - -ExUnit.start(exclude: exclude) +ExUnit.start() From c889ff9ee312d1f4f13516d73441d80ed85116b0 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:27:43 +0300 Subject: [PATCH 09/11] eh --- test/ecto/integration/json_test.exs | 2 +- test/support/migrations.ex | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 7556c919..e3586b7b 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -7,7 +7,7 @@ defmodule Ecto.Integration.JsonTest do alias Ecto.Integration.TestRepo alias EctoClickHouse.Integration.Setting - @tag :skip + @tag skip: true test "serializes json correctly" do # Insert a record purposefully with atoms as the map key. We are going to # verify later they were coerced into strings. diff --git a/test/support/migrations.ex b/test/support/migrations.ex index 0a837d66..8de3a861 100644 --- a/test/support/migrations.ex +++ b/test/support/migrations.ex @@ -189,8 +189,9 @@ defmodule EctoClickHouse.Integration.Migration do add :z, :Float64 end - create table(:settings, primary_key: false, engine: "MergeTree", options: "order by tuple()") do - add :properties, :JSON - end + # TODO + # create table(:settings, primary_key: false, engine: "MergeTree") do + # add :properties, :JSON + # end end end From 4c2230855163fdc2093e4229e4234c94ea521701 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:29:20 +0300 Subject: [PATCH 10/11] update changelog --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b205bc..5eef9351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,7 @@ ## Unreleased -- add [Time](https://clickhouse.com/docs/sql-reference/data-types/time) and [Time64](https://clickhouse.com/docs/sql-reference/data-types/time64) types support -- add [Variant](https://clickhouse.com/docs/sql-reference/data-types/variant) type support -- add [JSON](https://clickhouse.com/docs/sql-reference/data-types/newjson) type support +- update Ch to [v0.5.x](https://github.com/plausible/ch/blob/master/CHANGELOG.md#050-2025-07-17) which adds Time, Variant, and JSON support https://github.com/plausible/ecto_ch/pull/233 ## 0.7.1 (2025-07-07) From 2726238a3321c168d287fe819b66babeff10cf47 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 17 Jul 2025 20:31:32 +0300 Subject: [PATCH 11/11] enable_json_type: 1 --- test/test_helper.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_helper.exs b/test/test_helper.exs index 594202dd..3de8eb77 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -28,6 +28,7 @@ alias Ecto.Integration.TestRepo Application.put_env(:ecto_ch, TestRepo, adapter: Ecto.Adapters.ClickHouse, database: "ecto_ch_test", + settings: [enable_json_type: 1], show_sensitive_data_on_connection_error: true )