#!/usr/bin/env bash
# Tags: no-parallel, disabled
#test how MSTG vector index support final, brute force search is required
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

echo "Create table and insert data"
clickhouse client -q "DROP TABLE IF EXISTS default.arxiv_small_payload_test SYNC;"
clickhouse client -q "CREATE TABLE default.arxiv_small_payload_test \
                      ( \
                          id UInt32, \
                          vector Array(Float32), \
                          update_date_ts Int32, \
                          label LowCardinality(String),\
                          need_update UInt64 DEFAULT rand64(),\
                          flag Int32, \
                          VECTOR INDEX mstg_index vector TYPE MSTG(), \
                          CONSTRAINT check_length CHECK length(vector) = 384 \
                      ) \
                      ENGINE = ReplacingMergeTree \
                      ORDER BY id \
                      SETTINGS min_rows_to_build_vector_index=0, default_mstg_disk_mode = 1;"
clickhouse client -q "INSERT INTO default.arxiv_small_payload_test FORMAT Parquet" < "$CURDIR"/helpers/arxiv_small_payload.parquet

echo "Stop merges on test table to avoid redundant values are deleted when merged"
clickhouse client -q "SYSTEM STOP MERGES default.arxiv_small_payload_test"

echo "Insert same values into the table again"
clickhouse client -q "INSERT INTO default.arxiv_small_payload_test FORMAT Parquet" < "$CURDIR"/helpers/arxiv_small_payload.parquet

echo "Select without final there are redundant values"
clickhouse client -q "SELECT id,distance(vector,     \
                                          [-0.055295628, 0.015854683, -0.036837257, -0.003345381, 0.024053926,-0.04778976, -0.037346177,           \
                                          0.102654815, -0.01934478, -0.03958162,0.02728625, -0.10185599, -0.035011634, -0.038921114, 0.008217462,   \
                                          0.020245867, -0.07048218, -0.044691786, -0.007060565, -0.044124216,-0.06772715, 0.004919869, 0.029127918, \
                                          0.032422528, 0.06499404, -0.061767742, 0.01100607, 0.0215292, 0.07672191, 0.016523372, 0.03484405,        \
                                          0.028356323, -0.04475319, 0.026527757, 0.040244542, 0.008821562, -0.028348915, 0.0162884, -0.0031549763,  \
                                          0.0070102788,0.019138565, 0.011111138, 0.0153474705, -0.04989895, 0.06862658,0.015160324, -0.039039925,    \
                                          -0.023156717, -0.0018436415, 0.07972706,0.054872505, -0.039124195, 0.015175216, 0.109094836, -0.028682204, \
                                          0.043602105, 0.006524335, -0.046220522, 0.041714318, 0.03218298, -0.095331274, -0.017587908, -0.064463064, \
                                          -0.047254775, 0.053172547, 0.018260969, -0.058546096, 0.034314197, -0.006507517, 0.034726545, 0.01774933,  \
                                          0.039006703, -0.043406, -0.099716105, -0.030374791, 0.04365915, 0.085398644, -0.008313301, -0.05162726,    \
                                          -0.005699648,0.02844129, -0.025657263, -0.08825004, -0.08967642, -0.045910046, 0.0045650997, -0.027187336, \
                                          0.028064601, 0.07554649, -0.023668263, 0.047619164, -0.12343444, -0.03857834, -0.037431687, -0.02385608,    \
                                          0.11220963, 0.050074816, -0.066862, 0.021361103, -0.012573237, -0.008416121, 0.00750025,                    \
                                          0.0022115323, -0.009315773, -0.035588525, 0.034235455, 0.039341453, 0.087452024, -0.013766584, -0.04357595, \
                                          0.0825073, -0.007054246, 0.022904608, -0.0772567, -0.06892884,0.0063113775, 0.0003517152, \
                                          -0.060345545, 0.042075112, -0.07791268, 0.04434326, 0.0713569, 0.04569407, 0.05951997, -0.0035746284, \
                                          -0.059430942, -0.025838958, 2.1299363e-34, 0.05980308, -0.037984055,0.030723697, 0.0010010189,-0.10851818, \
                                          -0.0054221386, 0.02176743,-0.0188837, 0.021196933, -0.045663923, -0.08055672, -0.06136938, 0.026402136, \
                                          -0.048221793, -0.027296552, -0.039678443, 0.026098453,0.054824956, -0.043044332, 0.10449089, -0.062232412, \
                                          -0.016554844, -0.03625609, 0.037586074, 0.06334312, 0.10280571, -0.04775897,-0.030672317, -0.14309792, \
                                          0.03410216, 0.03855362, 0.07698783,-0.017047182, 0.062645525, 0.04921719, 0.02410303, -0.024767602, \
                                          -0.013013182, 0.056870036, 0.016483437, 0.047808852, 0.059962235,-0.017383907, -0.06691343, -0.037882853, \
                                          -0.047925238, 0.06791053,-0.04847118, 0.039308827, 0.048673313, 0.022925807, 0.06488089, \
                                          0.027612606, -0.061257493, 0.05471414, -0.032746278, 0.049911287,0.018671395, 0.10582697, 0.07604196, \
                                          -0.012432286, 0.06270822, -0.029083828, 0.047691606, -0.08693231, 0.068120636, -0.050220642, \
                                          -0.00010127072, 0.025818268, 0.0170994, -0.070562474, 0.0067966543, 0.013675752, -0.08680923, \
                                          0.10071713, 0.021825213, -0.05014523, 0.0042276313, 0.0565159, 0.024533078, 0.0050824382, 0.022231502, \
                                          -0.037915085, -0.04651563, 0.00917099, -0.040879995, -0.027302977, 0.046540774, -0.037252087, 0.02816566,\
                                          -0.03047872, 0.014613939, 0.046433326, -0.008224052, -0.002935053, -1.1268988e-33, -0.012401757, 0.01965621,\
                                          -0.017485727, -0.05897598, -0.11040468, -0.011932528, 0.006055703, -0.023816207, -0.013154525, 0.10173752, \
                                          -0.016655331, 0.042105705, -0.11464109, -0.041875884, -0.022044638, 0.048247352, 0.03640561, 0.0040184394,\
                                          0.06425491, -0.037121333, 0.07473459, 0.026252378, 0.018779451, -0.025458265, 0.008264, -0.0359528, \
                                          0.033228174, -0.07618842, 0.09241946, -0.064928636, -0.10551641, -0.09366349, 0.06189027, 0.12622999, \
                                          0.010030613, -0.02896201, 0.04284317, 0.015480006, 0.08918644, -0.029814815, 0.06112921, 0.010208654, \
                                          -0.0013521541, 0.01365302, -0.022887312, -0.0016102322, -0.03124973, 0.033850458, 0.02826661, -0.048791766,\
                                          -0.024780342, -0.016789403, 0.08176545, 0.046140786, -0.004785436, -0.059706815, -0.0024006034, 0.0053002196,\
                                          -0.003702689, -0.10019132, 0.02624131, -0.117855445, 0.009382043, -0.08231825, 0.012030045, 0.025364028, \
                                          0.0044562225, 0.031453494, -0.035732944, -0.039166465, 0.0091929855, -0.017792577, 0.13809888, -0.08741632, \
                                          -0.041251093, -0.05012265, 0.10124762, -0.07415356, 0.14692159, -0.03423576, -0.05604226, 0.007949296, -0.011579418, \
                                          -0.038550507, 0.029214397, 0.027016807, -0.04023333, 0.046516035, -0.036016844, -0.103562735, -0.055736754, 0.00892993, \
                                          0.10770055, 0.059727214, 0.081321836, -1.809326e-8, 0.07255331, -0.079738356, -0.043842167, 0.045095917, 0.042764075,\
                                          0.01408105, -0.03449506, -0.009586448, -0.004472831, 0.044103537, 0.0844983, -0.03326161, 0.04180339, \
                                          -0.021692377,0.011167308, -0.009195055, -0.0089334445, 0.034694556, -0.036402185, 0.0069375313, 0.048388917,\
                                          0.008733594,0.086927965, -0.008517176, -0.08339961, 0.031806137, -0.0018051409, -0.01761339, 0.01426462, \
                                          -0.08703955,-0.020425389, -0.06386625, -0.059769318, 0.008721534, 0.03276776, -0.0016394121, -0.06210337, \
                                          0.059858195, 0.14972606, 0.096824124, -0.057291567, 0.035870332, 0.015777508, 0.027756158, -0.042395603, \
                                          0.060652588, -0.029295119, -0.0024896055, -0.06265461, -0.02046819, -0.031429734, -0.018331336, 0.03811979, \
                                          -0.0147354035,-0.023692103, -0.0074687675, -0.027361125, 0.0038417606, 0.006437688, -0.052888975, 0.111916244, \
                                          0.03241323,-0.06350597, 0.034687404] \
                                         ) AS dist \
                      FROM arxiv_small_payload_test \
                      ORDER BY dist ASC \
                      LIMIT 10 \
                      SETTINGS enable_brute_force_vector_search=1"

echo "Select with final = 1 there are no redundant values"
clickhouse client -q "SELECT id,distance(vector,     \
                                          [-0.055295628, 0.015854683, -0.036837257, -0.003345381, 0.024053926,-0.04778976, -0.037346177,           \
                                          0.102654815, -0.01934478, -0.03958162,0.02728625, -0.10185599, -0.035011634, -0.038921114, 0.008217462,   \
                                          0.020245867, -0.07048218, -0.044691786, -0.007060565, -0.044124216,-0.06772715, 0.004919869, 0.029127918, \
                                          0.032422528, 0.06499404, -0.061767742, 0.01100607, 0.0215292, 0.07672191, 0.016523372, 0.03484405,        \
                                          0.028356323, -0.04475319, 0.026527757, 0.040244542, 0.008821562, -0.028348915, 0.0162884, -0.0031549763,  \
                                          0.0070102788,0.019138565, 0.011111138, 0.0153474705, -0.04989895, 0.06862658,0.015160324, -0.039039925,    \
                                          -0.023156717, -0.0018436415, 0.07972706,0.054872505, -0.039124195, 0.015175216, 0.109094836, -0.028682204, \
                                          0.043602105, 0.006524335, -0.046220522, 0.041714318, 0.03218298, -0.095331274, -0.017587908, -0.064463064, \
                                          -0.047254775, 0.053172547, 0.018260969, -0.058546096, 0.034314197, -0.006507517, 0.034726545, 0.01774933,  \
                                          0.039006703, -0.043406, -0.099716105, -0.030374791, 0.04365915, 0.085398644, -0.008313301, -0.05162726,    \
                                          -0.005699648,0.02844129, -0.025657263, -0.08825004, -0.08967642, -0.045910046, 0.0045650997, -0.027187336, \
                                          0.028064601, 0.07554649, -0.023668263, 0.047619164, -0.12343444, -0.03857834, -0.037431687, -0.02385608,    \
                                          0.11220963, 0.050074816, -0.066862, 0.021361103, -0.012573237, -0.008416121, 0.00750025,                    \
                                          0.0022115323, -0.009315773, -0.035588525, 0.034235455, 0.039341453, 0.087452024, -0.013766584, -0.04357595, \
                                          0.0825073, -0.007054246, 0.022904608, -0.0772567, -0.06892884,0.0063113775, 0.0003517152, \
                                          -0.060345545, 0.042075112, -0.07791268, 0.04434326, 0.0713569, 0.04569407, 0.05951997, -0.0035746284, \
                                          -0.059430942, -0.025838958, 2.1299363e-34, 0.05980308, -0.037984055,0.030723697, 0.0010010189,-0.10851818, \
                                          -0.0054221386, 0.02176743,-0.0188837, 0.021196933, -0.045663923, -0.08055672, -0.06136938, 0.026402136, \
                                          -0.048221793, -0.027296552, -0.039678443, 0.026098453,0.054824956, -0.043044332, 0.10449089, -0.062232412, \
                                          -0.016554844, -0.03625609, 0.037586074, 0.06334312, 0.10280571, -0.04775897,-0.030672317, -0.14309792, \
                                          0.03410216, 0.03855362, 0.07698783,-0.017047182, 0.062645525, 0.04921719, 0.02410303, -0.024767602, \
                                          -0.013013182, 0.056870036, 0.016483437, 0.047808852, 0.059962235,-0.017383907, -0.06691343, -0.037882853, \
                                          -0.047925238, 0.06791053,-0.04847118, 0.039308827, 0.048673313, 0.022925807, 0.06488089, \
                                          0.027612606, -0.061257493, 0.05471414, -0.032746278, 0.049911287,0.018671395, 0.10582697, 0.07604196, \
                                          -0.012432286, 0.06270822, -0.029083828, 0.047691606, -0.08693231, 0.068120636, -0.050220642, \
                                          -0.00010127072, 0.025818268, 0.0170994, -0.070562474, 0.0067966543, 0.013675752, -0.08680923, \
                                          0.10071713, 0.021825213, -0.05014523, 0.0042276313, 0.0565159, 0.024533078, 0.0050824382, 0.022231502, \
                                          -0.037915085, -0.04651563, 0.00917099, -0.040879995, -0.027302977, 0.046540774, -0.037252087, 0.02816566,\
                                          -0.03047872, 0.014613939, 0.046433326, -0.008224052, -0.002935053, -1.1268988e-33, -0.012401757, 0.01965621,\
                                          -0.017485727, -0.05897598, -0.11040468, -0.011932528, 0.006055703, -0.023816207, -0.013154525, 0.10173752, \
                                          -0.016655331, 0.042105705, -0.11464109, -0.041875884, -0.022044638, 0.048247352, 0.03640561, 0.0040184394,\
                                          0.06425491, -0.037121333, 0.07473459, 0.026252378, 0.018779451, -0.025458265, 0.008264, -0.0359528, \
                                          0.033228174, -0.07618842, 0.09241946, -0.064928636, -0.10551641, -0.09366349, 0.06189027, 0.12622999, \
                                          0.010030613, -0.02896201, 0.04284317, 0.015480006, 0.08918644, -0.029814815, 0.06112921, 0.010208654, \
                                          -0.0013521541, 0.01365302, -0.022887312, -0.0016102322, -0.03124973, 0.033850458, 0.02826661, -0.048791766,\
                                          -0.024780342, -0.016789403, 0.08176545, 0.046140786, -0.004785436, -0.059706815, -0.0024006034, 0.0053002196,\
                                          -0.003702689, -0.10019132, 0.02624131, -0.117855445, 0.009382043, -0.08231825, 0.012030045, 0.025364028, \
                                          0.0044562225, 0.031453494, -0.035732944, -0.039166465, 0.0091929855, -0.017792577, 0.13809888, -0.08741632, \
                                          -0.041251093, -0.05012265, 0.10124762, -0.07415356, 0.14692159, -0.03423576, -0.05604226, 0.007949296, -0.011579418, \
                                          -0.038550507, 0.029214397, 0.027016807, -0.04023333, 0.046516035, -0.036016844, -0.103562735, -0.055736754, 0.00892993, \
                                          0.10770055, 0.059727214, 0.081321836, -1.809326e-8, 0.07255331, -0.079738356, -0.043842167, 0.045095917, 0.042764075,\
                                          0.01408105, -0.03449506, -0.009586448, -0.004472831, 0.044103537, 0.0844983, -0.03326161, 0.04180339, \
                                          -0.021692377,0.011167308, -0.009195055, -0.0089334445, 0.034694556, -0.036402185, 0.0069375313, 0.048388917,\
                                          0.008733594,0.086927965, -0.008517176, -0.08339961, 0.031806137, -0.0018051409, -0.01761339, 0.01426462, \
                                          -0.08703955,-0.020425389, -0.06386625, -0.059769318, 0.008721534, 0.03276776, -0.0016394121, -0.06210337, \
                                          0.059858195, 0.14972606, 0.096824124, -0.057291567, 0.035870332, 0.015777508, 0.027756158, -0.042395603, \
                                          0.060652588, -0.029295119, -0.0024896055, -0.06265461, -0.02046819, -0.031429734, -0.018331336, 0.03811979, \
                                          -0.0147354035,-0.023692103, -0.0074687675, -0.027361125, 0.0038417606, 0.006437688, -0.052888975, 0.111916244, \
                                          0.03241323,-0.06350597, 0.034687404] \
                                         ) AS dist \
                      FROM arxiv_small_payload_test \
                      ORDER BY dist ASC \
                      LIMIT 10 \
                      SETTINGS final=1, enable_brute_force_vector_search=1"
echo "Insert same values into the table again"
clickhouse client -q "INSERT INTO default.arxiv_small_payload_test FORMAT Parquet" < "$CURDIR"/helpers/arxiv_small_payload.parquet

echo "Select with final and two_stage search enabled"
clickhouse client -q "SELECT id,distance(vector,     \
                                          [-0.055295628, 0.015854683, -0.036837257, -0.003345381, 0.024053926,-0.04778976, -0.037346177,           \
                                          0.102654815, -0.01934478, -0.03958162,0.02728625, -0.10185599, -0.035011634, -0.038921114, 0.008217462,   \
                                          0.020245867, -0.07048218, -0.044691786, -0.007060565, -0.044124216,-0.06772715, 0.004919869, 0.029127918, \
                                          0.032422528, 0.06499404, -0.061767742, 0.01100607, 0.0215292, 0.07672191, 0.016523372, 0.03484405,        \
                                          0.028356323, -0.04475319, 0.026527757, 0.040244542, 0.008821562, -0.028348915, 0.0162884, -0.0031549763,  \
                                          0.0070102788,0.019138565, 0.011111138, 0.0153474705, -0.04989895, 0.06862658,0.015160324, -0.039039925,    \
                                          -0.023156717, -0.0018436415, 0.07972706,0.054872505, -0.039124195, 0.015175216, 0.109094836, -0.028682204, \
                                          0.043602105, 0.006524335, -0.046220522, 0.041714318, 0.03218298, -0.095331274, -0.017587908, -0.064463064, \
                                          -0.047254775, 0.053172547, 0.018260969, -0.058546096, 0.034314197, -0.006507517, 0.034726545, 0.01774933,  \
                                          0.039006703, -0.043406, -0.099716105, -0.030374791, 0.04365915, 0.085398644, -0.008313301, -0.05162726,    \
                                          -0.005699648,0.02844129, -0.025657263, -0.08825004, -0.08967642, -0.045910046, 0.0045650997, -0.027187336, \
                                          0.028064601, 0.07554649, -0.023668263, 0.047619164, -0.12343444, -0.03857834, -0.037431687, -0.02385608,    \
                                          0.11220963, 0.050074816, -0.066862, 0.021361103, -0.012573237, -0.008416121, 0.00750025,                    \
                                          0.0022115323, -0.009315773, -0.035588525, 0.034235455, 0.039341453, 0.087452024, -0.013766584, -0.04357595, \
                                          0.0825073, -0.007054246, 0.022904608, -0.0772567, -0.06892884,0.0063113775, 0.0003517152, \
                                          -0.060345545, 0.042075112, -0.07791268, 0.04434326, 0.0713569, 0.04569407, 0.05951997, -0.0035746284, \
                                          -0.059430942, -0.025838958, 2.1299363e-34, 0.05980308, -0.037984055,0.030723697, 0.0010010189,-0.10851818, \
                                          -0.0054221386, 0.02176743,-0.0188837, 0.021196933, -0.045663923, -0.08055672, -0.06136938, 0.026402136, \
                                          -0.048221793, -0.027296552, -0.039678443, 0.026098453,0.054824956, -0.043044332, 0.10449089, -0.062232412, \
                                          -0.016554844, -0.03625609, 0.037586074, 0.06334312, 0.10280571, -0.04775897,-0.030672317, -0.14309792, \
                                          0.03410216, 0.03855362, 0.07698783,-0.017047182, 0.062645525, 0.04921719, 0.02410303, -0.024767602, \
                                          -0.013013182, 0.056870036, 0.016483437, 0.047808852, 0.059962235,-0.017383907, -0.06691343, -0.037882853, \
                                          -0.047925238, 0.06791053,-0.04847118, 0.039308827, 0.048673313, 0.022925807, 0.06488089, \
                                          0.027612606, -0.061257493, 0.05471414, -0.032746278, 0.049911287,0.018671395, 0.10582697, 0.07604196, \
                                          -0.012432286, 0.06270822, -0.029083828, 0.047691606, -0.08693231, 0.068120636, -0.050220642, \
                                          -0.00010127072, 0.025818268, 0.0170994, -0.070562474, 0.0067966543, 0.013675752, -0.08680923, \
                                          0.10071713, 0.021825213, -0.05014523, 0.0042276313, 0.0565159, 0.024533078, 0.0050824382, 0.022231502, \
                                          -0.037915085, -0.04651563, 0.00917099, -0.040879995, -0.027302977, 0.046540774, -0.037252087, 0.02816566,\
                                          -0.03047872, 0.014613939, 0.046433326, -0.008224052, -0.002935053, -1.1268988e-33, -0.012401757, 0.01965621,\
                                          -0.017485727, -0.05897598, -0.11040468, -0.011932528, 0.006055703, -0.023816207, -0.013154525, 0.10173752, \
                                          -0.016655331, 0.042105705, -0.11464109, -0.041875884, -0.022044638, 0.048247352, 0.03640561, 0.0040184394,\
                                          0.06425491, -0.037121333, 0.07473459, 0.026252378, 0.018779451, -0.025458265, 0.008264, -0.0359528, \
                                          0.033228174, -0.07618842, 0.09241946, -0.064928636, -0.10551641, -0.09366349, 0.06189027, 0.12622999, \
                                          0.010030613, -0.02896201, 0.04284317, 0.015480006, 0.08918644, -0.029814815, 0.06112921, 0.010208654, \
                                          -0.0013521541, 0.01365302, -0.022887312, -0.0016102322, -0.03124973, 0.033850458, 0.02826661, -0.048791766,\
                                          -0.024780342, -0.016789403, 0.08176545, 0.046140786, -0.004785436, -0.059706815, -0.0024006034, 0.0053002196,\
                                          -0.003702689, -0.10019132, 0.02624131, -0.117855445, 0.009382043, -0.08231825, 0.012030045, 0.025364028, \
                                          0.0044562225, 0.031453494, -0.035732944, -0.039166465, 0.0091929855, -0.017792577, 0.13809888, -0.08741632, \
                                          -0.041251093, -0.05012265, 0.10124762, -0.07415356, 0.14692159, -0.03423576, -0.05604226, 0.007949296, -0.011579418, \
                                          -0.038550507, 0.029214397, 0.027016807, -0.04023333, 0.046516035, -0.036016844, -0.103562735, -0.055736754, 0.00892993, \
                                          0.10770055, 0.059727214, 0.081321836, -1.809326e-8, 0.07255331, -0.079738356, -0.043842167, 0.045095917, 0.042764075,\
                                          0.01408105, -0.03449506, -0.009586448, -0.004472831, 0.044103537, 0.0844983, -0.03326161, 0.04180339, \
                                          -0.021692377,0.011167308, -0.009195055, -0.0089334445, 0.034694556, -0.036402185, 0.0069375313, 0.048388917,\
                                          0.008733594,0.086927965, -0.008517176, -0.08339961, 0.031806137, -0.0018051409, -0.01761339, 0.01426462, \
                                          -0.08703955,-0.020425389, -0.06386625, -0.059769318, 0.008721534, 0.03276776, -0.0016394121, -0.06210337, \
                                          0.059858195, 0.14972606, 0.096824124, -0.057291567, 0.035870332, 0.015777508, 0.027756158, -0.042395603, \
                                          0.060652588, -0.029295119, -0.0024896055, -0.06265461, -0.02046819, -0.031429734, -0.018331336, 0.03811979, \
                                          -0.0147354035,-0.023692103, -0.0074687675, -0.027361125, 0.0038417606, 0.006437688, -0.052888975, 0.111916244, \
                                          0.03241323,-0.06350597, 0.034687404] \
                                         ) AS dist \
                      FROM arxiv_small_payload_test \
                      ORDER BY dist ASC \
                      LIMIT 10 \
                      SETTINGS final=1, enable_brute_force_vector_search=1, two_stage_search_option=2"

clickhouse client -q "DROP TABLE IF EXISTS default.arxiv_small_payload_test SYNC;"


