Materialized views are not triggered in test env.

[tee0zed]

I have very cryptic one also.

In test env, using transactional_fixtures or not, materialized views never triggered. I don't see any connections with Rails.env and Clickhouse, but in dev i have my SummingMergeTree table filling with MV described with TO statement, but in test environment i have only first, MergeTree table been filling with records, but views seems to be dead. Clickhouse logs seems legit, no errors.

class CreateEntries < ActiveRecord::Migration[7.1]
  def change
    create_table 'entries', id: false, null: false,
      options: 'MergeTree() PARTITION BY (partner_id, toYYYYMM(created_at)) ORDER BY (wallet_id, created_at)', force: :cascade do |t|
        t.uuid 'id', default: -> { 'generateUUIDv4()' }, null: false
        t.uuid 'partner_id', null: false
        t.uuid 'wallet_id', null: false
        t.uuid 'transaction_id', null: false
        t.string 'desc', limit: 128
        t.column 'available', 'Int32', null: false
        t.column 'hold', 'Int32', null: false
        t.column 'type', 'LowCardinality(String)', null: false
        t.column 'created_at', 'DateTime64', default: -> { "CAST(now(), 'DateTime64')" }, null: false
      end
  end
end

class AddWalletsSummingMergeTree < ActiveRecord::Migration[7.1]
  def change
    create_table 'summing_entries', id: false, null: false, options: 'SummingMergeTree((available, hold)) ORDER BY wallet_id', force: :cascade do |t|
      t.uuid 'partner_id', null: false
      t.uuid 'wallet_id', null: false
      t.column 'available', 'Int32', null: false
      t.column 'hold', 'Int32', null: false
      t.column 'created_at', 'DateTime64', null: false
    end
  end
end

class AddSummingEntriesView < ActiveRecord::Migration[7.1]
  def up
    execute <<~SQL.squish
      CREATE MATERIALIZED VIEW IF NOT EXISTS summing_entries_mv TO summing_entries AS
      SELECT
        partner_id,
        wallet_id,
        available,
        hold,
        created_at
      FROM entries
    SQL
  end

  def down
    drop_table 'summing_entries_mv' if table_exists? 'summing_entries_mv'
  end
end

This is my migrations

This is how I insert records in test

module Wallets
  class Entry < ClickHouseRecord
    def self.batch_insert(entries)
      insert_all(entries.map(&:attributes)) # rubocop:disable Rails/SkipsModelValidations
    end
  end 
end 


RSpec.describe Wallet, clickhouse: true do
  let(:wallet) { create(:wallet, hold: 100, available: 100) }

  before do
    Wallets::Entry.batch_insert(
      [
        build(:entry, wallet:, available: 0, hold: 100, type: 'hold', desc: 'Hold'),
        build(:entry, wallet:, available: 100, hold: 0, type: 'deposit', desc: 'Hold')
      ]
    )
  end
.....

This is hooks configuration that i use

RSpec.configure do |config|
  config.before(:each, clickhouse: true) do
    RSpec.configuration.use_transactional_fixtures = false

    ClickHouseRecord.establish_connection(:clickhouse)
    ClickHouseRecord.connection.execute('TRUNCATE TABLE entries')
    ClickHouseRecord.connection.execute('TRUNCATE TABLE summing_entries')
    ClickHouseRecord.connection.execute('TRUNCATE TABLE summing_entries_mv')
  end

  config.after(:each, clickhouse: true) do
    RSpec.configuration.use_transactional_fixtures = true

    ApplicationRecord.connection.reconnect!
  end
end

I also tried to set .use_transactional_fixtures = false in base rspec config

Have you seen this before?

[tee0zed]

Yeah definetely test env, reproduced in RAILS_ENV=test rails c

[senid231]

for testing clickhouse there is a better approach

module TestClickhouseHelper
  class InsertCounter
    include Singleton

    attr_reader :data

    def initialize
      reset
    end

    def reset
      @data = {}
    end

    def qty
      @data.values.sum
    end

    def active_klasses
      @data.select { |_, qty| qty.positive? }.keys.map(&:constantize)
    end

    def increment(klass)
      @data[klass.to_s] ||= 0
      @data[klass.to_s] += 1
    end
  end

  module ExampleGroups
    # Do not wrap clickhouse connections in transactions because it does not support transactions.
    # @see ActiveRecord::TestFixtures#enlist_fixture_connections
    def enlist_fixture_connections
      connections = super
      connections.reject { |conn| conn.is_a?(ActiveRecord::ConnectionAdapters::ClickhouseAdapter) }
    end
  end
end

ActiveRecord::TestFixtures.prepend TestClickhouseHelper::ExampleGroups

clickhouse_classes = ClickHouseRecord.descendants
clickhouse_classes.each do |klass|
  klass.after_create { TestClickhouseHelper::InsertCounter.instance.increment(self.class) }
end

RSpec.configure do |config|
  config.after do
    # truncate clickhouse tables only if they have inserts
    TestClickhouseHelper::InsertCounter.instance.active_klasses.each do |klass|
      ClickHouseRecord.connection.execute("TRUNCATE TABLE #{klass.table_name}")
    end
    TestClickhouseHelper::InsertCounter.instance.reset
  end
end