Skip to content

Commit 3651278

Browse files
gatesngatesn
authored
Fix strategy (#2785)
Clickbench file now looks like this: ``` 0: 8..554000 (len=553992, alignment=8) - .Title.data.[0] 1: 554000..1020752 (len=466752, alignment=8) - .Title.data.[1] 2: 1020752..1483504 (len=462752, alignment=8) - .Title.data.[2] 3: 1483504..1941472 (len=457968, alignment=8) - .Title.data.[3] 4: 1941472..2401608 (len=460136, alignment=8) - .Title.data.[4] 5: 2401608..3319144 (len=917536, alignment=8) - .URL.data.[0] 6: 3319144..3572392 (len=253248, alignment=8) - .URL.data.[1] 7: 3572392..3833672 (len=261280, alignment=8) - .URL.data.[2] 8: 3833672..4092936 (len=259264, alignment=8) - .URL.data.[3] 9: 4092936..4355952 (len=263016, alignment=8) - .URL.data.[4] 10: 4355952..4626408 (len=270456, alignment=8) - .URL.data.[5] 11: 4626408..5500520 (len=874112, alignment=8) - .Referer.data.[0] 12: 5500520..5758720 (len=258200, alignment=8) - .Referer.data.[1] 13: 5758720..6020656 (len=261936, alignment=8) - .Referer.data.[2] 14: 6020656..6278208 (len=257552, alignment=8) - .Referer.data.[3] 15: 6278208..6539584 (len=261376, alignment=8) - .Referer.data.[4] 16: 6539584..6834264 (len=294680, alignment=8) - .Referer.data.[5] 17: 6834264..7098128 (len=263864, alignment=8) - .Referer.data.[6] 18: 7098128..7241800 (len=143672, alignment=8) - .URL.data.[6] 19: 7241800..7382176 (len=140376, alignment=8) - .URL.data.[7] 20: 7382176..7685736 (len=303560, alignment=8) - .URL.data.[8] 21: 7685736..7972952 (len=287216, alignment=8) - .URL.data.[9] 22: 7972952..8289376 (len=316424, alignment=8) - .URL.data.[10] 23: 8289376..8442456 (len=153080, alignment=8) - .URL.data.[11] 24: 8442456..8704904 (len=262448, alignment=8) - .URL.data.[12] 25: 8704904..8841024 (len=136120, alignment=8) - .URL.data.[13] 26: 8841024..9131040 (len=290016, alignment=8) - .URL.data.[14] 27: 9131040..9451176 (len=320136, alignment=8) - .URL.data.[15] 28: 9451176..9765968 (len=314792, alignment=8) - .URL.data.[16] 29: 9765968..10054680 (len=288712, alignment=8) - .Referer.data.[7] 30: 10054680..10323584 (len=268904, alignment=8) - .Referer.data.[8] 31: 10323584..10618744 (len=295160, alignment=8) - .Referer.data.[9] 32: 10618744..10763344 (len=144600, alignment=8) - .Referer.data.[10] 33: 10763344..11055592 (len=292248, alignment=8) - .Referer.data.[11] 34: 11055592..11487336 (len=431744, alignment=8) - .Referer.data.[12] 35: 11487336..11783224 (len=295888, alignment=8) - .Referer.data.[13] 36: 11783224..12060808 (len=277584, alignment=8) - .Referer.data.[14] 37: 12060808..12551704 (len=490896, alignment=8) - .Title.data.[5] 38: 12551704..13026512 (len=474808, alignment=8) - .Title.data.[6] 39: 13026512..13440104 (len=413592, alignment=8) - .Title.data.[7] 40: 13440104..13924904 (len=484800, alignment=8) - .Title.data.[8] 41: 13924904..14246400 (len=321496, alignment=8) - .Title.data.[9] 42: 14246400..14796776 (len=550376, alignment=8) - .Title.data.[10] 43: 14796776..14965736 (len=168960, alignment=8) - .URL.data.[17] 44: 14965736..15518976 (len=553240, alignment=8) - .URL.data.[18] 45: 15518976..15690960 (len=171984, alignment=8) - .URL.data.[19] 46: 15690960..15958104 (len=267144, alignment=8) - .URL.data.[20] 47: 15958104..16303400 (len=345296, alignment=8) - .URL.data.[21] 48: 16303400..16583728 (len=280328, alignment=8) - .URL.data.[22] 49: 16583728..16748824 (len=165096, alignment=8) - .URL.data.[23] 50: 16748824..17272232 (len=523408, alignment=8) - .URL.data.[24] 51: 17272232..17432128 (len=159896, alignment=8) - .Referer.data.[15] 52: 17432128..17634040 (len=201912, alignment=8) - .Referer.data.[16] 53: 17634040..18026056 (len=392016, alignment=8) - .Referer.data.[17] 54: 18026056..18301288 (len=275232, alignment=8) - .Referer.data.[18] 55: 18301288..18562208 (len=260920, alignment=8) - .Referer.data.[19] 56: 18562208..18721568 (len=159360, alignment=8) - .Referer.data.[20] 57: 18721568..18923408 (len=201840, alignment=8) - .Referer.data.[21] 58: 18923408..19316264 (len=392856, alignment=8) - .Referer.data.[22] 59: 19316264..19601296 (len=285032, alignment=8) - .Referer.data.[23] 60: 19601296..20042832 (len=441536, alignment=8) - .OriginalURL.data.[0] 61: 20042832..20194456 (len=151624, alignment=8) - .OriginalURL.data.[1] 62: 20194456..20461728 (len=267272, alignment=8) - .OriginalURL.data.[2] 63: 20461728..20499928 (len=38200, alignment=8) - .OriginalURL.data.[3] 64: 20499928..20645328 (len=145400, alignment=8) - .OriginalURL.data.[4] 65: 20645328..20937832 (len=292504, alignment=8) - .OriginalURL.data.[5] 66: 20937832..21299216 (len=361384, alignment=8) - .OriginalURL.data.[6] 67: 21299216..21453616 (len=154400, alignment=8) - .OriginalURL.data.[7] 68: 21453616..21782768 (len=329152, alignment=8) - .OriginalURL.data.[8] 69: 21782768..22136568 (len=353800, alignment=8) - .OriginalURL.data.[9] 70: 22136568..22308144 (len=171576, alignment=8) - .URL.data.[25] 71: 22308144..22582808 (len=274664, alignment=8) - .URL.data.[26] 72: 22582808..22912952 (len=330144, alignment=8) - .URL.data.[27] 73: 22912952..23212192 (len=299240, alignment=8) - .URL.data.[28] 74: 23212192..23373792 (len=161600, alignment=8) - .URL.data.[29] 75: 23373792..23861728 (len=487936, alignment=8) - .URL.data.[30] 76: 23861728..24033824 (len=172096, alignment=8) - .URL.data.[31] 77: 24033824..24343120 (len=309296, alignment=8) - .URL.data.[32] 78: 24343120..24641816 (len=298696, alignment=8) - .URL.data.[33] 79: 24641816..24915424 (len=273608, alignment=8) - .Referer.data.[24] 80: 24915424..25076368 (len=160944, alignment=8) - .Referer.data.[25] 81: 25076368..25279136 (len=202768, alignment=8) - .Referer.data.[26] 82: 25279136..25722416 (len=443280, alignment=8) - .Referer.data.[27] 83: 25722416..25988464 (len=266048, alignment=8) - .Referer.data.[28] 84: 25988464..26269040 (len=280576, alignment=8) - .Referer.data.[29] 85: 26269040..26431344 (len=162304, alignment=8) - .Referer.data.[30] 86: 26431344..26643520 (len=212176, alignment=8) - .Referer.data.[31] 87: 26643520..26925464 (len=281944, alignment=8) - .Referer.data.[32] 88: 26925464..27228744 (len=303280, alignment=8) - .URL.data.[34] 89: 27228744..27383880 (len=155136, alignment=8) - .URL.data.[35] 90: 27383880..27751576 (len=367696, alignment=8) - .URL.data.[36] 91: 27751576..27911704 (len=160128, alignment=8) - .URL.data.[37] 92: 27911704..28219760 (len=308056, alignment=8) - .URL.data.[38] 93: 28219760..28585736 (len=365976, alignment=8) - .URL.data.[39] 94: 28585736..28876320 (len=290584, alignment=8) - .URL.data.[40] 95: 28876320..29040472 (len=164152, alignment=8) - .URL.data.[41] 96: 29040472..29399648 (len=359176, alignment=8) - .URL.data.[42] 97: 29399648..29705520 (len=305872, alignment=8) - .Title.data.[11] 98: 29705520..30112416 (len=406896, alignment=8) - .Title.data.[12] 99: 30112416..30412320 (len=299904, alignment=8) - .Title.data.[13] 100: 30412320..30726248 (len=313928, alignment=8) - .Title.data.[14] 101: 30726248..31276688 (len=550440, alignment=8) - .Title.data.[15] 102: 31276688..31566528 (len=289840, alignment=8) - .Title.data.[16] 103: 31566528..31967216 (len=400688, alignment=8) - .Title.data.[17] 104: 31967216..32158984 (len=191768, alignment=8) - .OriginalURL.data.[10] 105: 32158984..32462512 (len=303528, alignment=8) - .OriginalURL.data.[11] 106: 32462512..32806688 (len=344176, alignment=8) - .OriginalURL.data.[12] 107: 32806688..32973176 (len=166488, alignment=8) - .OriginalURL.data.[13] 108: 32973176..33301944 (len=328768, alignment=8) - .OriginalURL.data.[14] 109: 33301944..33622560 (len=320616, alignment=8) - .OriginalURL.data.[15] 110: 33622560..33792752 (len=170192, alignment=8) - .OriginalURL.data.[16] 111: 33792752..34099984 (len=307232, alignment=8) - .OriginalURL.data.[17] 112: 34099984..34416912 (len=316928, alignment=8) - .OriginalURL.data.[18] 113: 34416912..34586216 (len=169304, alignment=8) - .URL.data.[43] 114: 34586216..34884624 (len=298408, alignment=8) - .URL.data.[44] 115: 34884624..35228200 (len=343576, alignment=8) - .URL.data.[45] 116: 35228200..35593056 (len=364856, alignment=8) - .URL.data.[46] 117: 35593056..35740592 (len=147536, alignment=8) - .URL.data.[47] 118: 35740592..36126408 (len=385816, alignment=8) - .URL.data.[48] 119: 36126408..36305944 (len=179536, alignment=8) - .URL.data.[49] 120: 36305944..36580272 (len=274328, alignment=8) - .URL.data.[50] 121: 36580272..36906400 (len=326128, alignment=8) - .URL.data.[51] 122: 36906400..37236704 (len=330304, alignment=8) - .Referer.data.[33] 123: 37236704..37509600 (len=272896, alignment=8) - .Referer.data.[34] 124: 37509600..37678400 (len=168800, alignment=8) - .Referer.data.[35] 125: 37678400..37898288 (len=219888, alignment=8) - .Referer.data.[36] 126: 37898288..38163808 (len=265520, alignment=8) - .Referer.data.[37] 127: 38163808..38478720 (len=314912, alignment=8) - .Referer.data.[38] 128: 38478720..38778704 (len=299984, alignment=8) - .Referer.data.[39] 129: 38778704..38942280 (len=163576, alignment=8) - .Referer.data.[40] 130: 38942280..39150864 (len=208584, alignment=8) - .Referer.data.[41] 131: 39150864..39404272 (len=253408, alignment=8) - .Referer.data.[42] 132: 39404272..39718680 (len=314408, alignment=8) - .URL.data.[52] 133: 39718680..39868856 (len=150176, alignment=8) - .URL.data.[53] 134: 39868856..40259016 (len=390160, alignment=8) - .URL.data.[54] 135: 40259016..40427200 (len=168184, alignment=8) - .URL.data.[55] 136: 40427200..40751120 (len=323920, alignment=8) - .URL.data.[56] 137: 40751120..41037816 (len=286696, alignment=8) - .URL.data.[57] 138: 41037816..41334264 (len=296448, alignment=8) - .URL.data.[58] 139: 41334264..41481440 (len=147176, alignment=8) - .URL.data.[59] 140: 41481440..41920320 (len=438880, alignment=8) - .URL.data.[60] 141: 41920320..42203904 (len=283584, alignment=8) - .Referer.data.[43] 142: 42203904..42485944 (len=282040, alignment=8) - .Referer.data.[44] 143: 42485944..42639632 (len=153688, alignment=8) - .Referer.data.[45] 144: 42639632..42839792 (len=200160, alignment=8) - .Referer.data.[46] 145: 42839792..43112840 (len=273048, alignment=8) - .Referer.data.[47] 146: 43112840..43358552 (len=245712, alignment=8) - .Referer.data.[48] 147: 43358552..43622352 (len=263800, alignment=8) - .Referer.data.[49] 148: 43622352..43772888 (len=150536, alignment=8) - .Referer.data.[50] 149: 43772888..43971688 (len=198800, alignment=8) - .Referer.data.[51] 150: 43971688..44241400 (len=269712, alignment=8) - .Referer.data.[52] 151: 44241400..44506376 (len=264976, alignment=8) - .Title.data.[18] 152: 44506376..44903096 (len=396720, alignment=8) - .Title.data.[19] 153: 44903096..45177200 (len=274104, alignment=8) - .Title.data.[20] 154: 45177200..45562688 (len=385488, alignment=8) - .Title.data.[21] 155: 45562688..45839736 (len=277048, alignment=8) - .Title.data.[22] 156: 45839736..46226312 (len=386576, alignment=8) - .Title.data.[23] 157: 46226312..46495144 (len=268832, alignment=8) - .Title.data.[24] 158: 46495144..46673896 (len=178752, alignment=8) - .OriginalURL.data.[19] 159: 46673896..47020584 (len=346688, alignment=8) - .OriginalURL.data.[20] 160: 47020584..47346016 (len=325432, alignment=8) - .OriginalURL.data.[21] 161: 47346016..47513312 (len=167296, alignment=8) - .OriginalURL.data.[22] 162: 47513312..47810016 (len=296704, alignment=8) - .OriginalURL.data.[23] 163: 47810016..48154288 (len=344272, alignment=8) - .OriginalURL.data.[24] 164: 48154288..48351860 (len=197572, alignment=8) - .OriginalURL.data.[25] 165: 48351864..48641336 (len=289472, alignment=8) - .OriginalURL.data.[26] 166: 48641336..49032104 (len=390768, alignment=8) - .OriginalURL.data.[27] 167: 49032104..49199400 (len=167296, alignment=8) - .URL.data.[61] 168: 49199400..49504680 (len=305280, alignment=8) - .URL.data.[62] 169: 49504680..49793840 (len=289160, alignment=8) - .URL.data.[63] 170: 49793840..50095368 (len=301528, alignment=8) - .URL.data.[64] 171: 50095368..50251272 (len=155904, alignment=8) - .URL.data.[65] 172: 50251272..50684152 (len=432880, alignment=8) - .URL.data.[66] 173: 50684152..50862960 (len=178808, alignment=8) - .URL.data.[67] 174: 50862960..51121864 (len=258904, alignment=8) - .URL.data.[68] 175: 51121864..51464688 (len=342824, alignment=8) - .URL.data.[69] 176: 51464688..51734024 (len=269336, alignment=8) - .Referer.data.[53] 177: 51734024..51992016 (len=257992, alignment=8) - .Referer.data.[54] 178: 51992016..52156328 (len=164312, alignment=8) - .Referer.data.[55] 179: 52156328..52347552 (len=191224, alignment=8) - .Referer.data.[56] 180: 52347552..52584048 (len=236496, alignment=8) - .Referer.data.[57] 181: 52584048..52906752 (len=322704, alignment=8) - .Referer.data.[58] 182: 52906752..53208592 (len=301840, alignment=8) - .Referer.data.[59] 183: 53208592..53371944 (len=163352, alignment=8) - .Referer.data.[60] 184: 53371944..53667416 (len=295472, alignment=8) - .Referer.data.[61] 185: 53667416..53942904 (len=275488, alignment=8) - .Referer.data.[62] 186: 53942904..54304432 (len=361528, alignment=8) - .URL.data.[70] 187: 54304432..54465056 (len=160624, alignment=8) - .URL.data.[71] 188: 54465056..54877240 (len=412184, alignment=8) - .URL.data.[72] 189: 54877240..55035560 (len=158320, alignment=8) - .URL.data.[73] 190: 55035560..55335416 (len=299856, alignment=8) - .URL.data.[74] 191: 55335416..55596856 (len=261440, alignment=8) - .URL.data.[75] 192: 55596856..55906544 (len=309688, alignment=8) - .URL.data.[76] 193: 55906544..56070040 (len=163496, alignment=8) - .URL.data.[77] 194: 56070040..56334904 (len=264864, alignment=8) - .URL.data.[78] 195: 56334904..56732736 (len=397832, alignment=8) - .Title.data.[25] 196: 56732736..57020272 (len=287536, alignment=8) - .Title.data.[26] 197: 57020272..57421952 (len=401680, alignment=8) - .Title.data.[27] 198: 57421952..57711992 (len=290040, alignment=8) - .Title.data.[28] 199: 57711992..58095960 (len=383968, alignment=8) - .Title.data.[29] 200: 58095960..58393032 (len=297072, alignment=8) - .Title.data.[30] 201: 58393032..58798976 (len=405944, alignment=8) - .Title.data.[31] 202: 58798976..59033416 (len=234440, alignment=8) - .Referer.data.[63] 203: 59033416..59312968 (len=279552, alignment=8) - .Referer.data.[64] 204: 59312968..59477672 (len=164704, alignment=8) - .Referer.data.[65] 205: 59477672..59886448 (len=408776, alignment=8) - .Referer.data.[66] 206: 59886448..60120248 (len=233800, alignment=8) - .Referer.data.[67] 207: 60120248..60412016 (len=291768, alignment=8) - .Referer.data.[68] 208: 60412016..60731576 (len=319560, alignment=8) - .Referer.data.[69] 209: 60731576..60899536 (len=167960, alignment=8) - .Referer.data.[70] 210: 60899536..61346992 (len=447456, alignment=8) - .Referer.data.[71] 211: 61346992..61505280 (len=158288, alignment=8) - .URL.data.[79] 212: 61505280..61769864 (len=264584, alignment=8) - .URL.data.[80] 213: 61769864..62103448 (len=333584, alignment=8) - .URL.data.[81] 214: 62103448..62446024 (len=342576, alignment=8) - .URL.data.[82] 215: 62446024..62607624 (len=161600, alignment=8) - .URL.data.[83] 216: 62607624..62895328 (len=287704, alignment=8) - .URL.data.[84] 217: 62895328..63062424 (len=167096, alignment=8) - .URL.data.[85] 218: 63062424..63386176 (len=323752, alignment=8) - .URL.data.[86] 219: 63386176..63680760 (len=294584, alignment=8) - .URL.data.[87] 220: 63680760..63983424 (len=302664, alignment=8) - .Title.data.[32] 221: 63983424..64396424 (len=413000, alignment=8) - .Title.data.[33] 222: 64396424..64702840 (len=306416, alignment=8) - .Title.data.[34] 223: 64702840..65118408 (len=415568, alignment=8) - .Title.data.[35] 224: 65118408..65444488 (len=326080, alignment=8) - .Title.data.[36] 225: 65444488..65853336 (len=408848, alignment=8) - .Title.data.[37] 226: 65853336..66331008 (len=477672, alignment=8) - .Title.data.[38] 227: 66331008..66599800 (len=268792, alignment=8) - .Referer.data.[72] 228: 66599800..66871920 (len=272120, alignment=8) - .Referer.data.[73] 229: 66871920..67182464 (len=310544, alignment=8) - .Referer.data.[74] 230: 67182464..67336528 (len=154064, alignment=8) - .Referer.data.[75] 231: 67336528..67787480 (len=450952, alignment=8) - .Referer.data.[76] 232: 67787480..68047280 (len=259800, alignment=8) - .Referer.data.[77] 233: 68047280..68353872 (len=306592, alignment=8) - .Referer.data.[78] 234: 68353872..68630912 (len=277040, alignment=8) - .Referer.data.[79] 235: 68630912..68785912 (len=155000, alignment=8) - .Referer.data.[80] 236: 68785912..76786124 (len=8000212, alignment=8) - .WatchID.data 237: 76786128..76911304 (len=125176, alignment=8) - .JavaEnable.data 238: 76911304..77386000 (len=474696, alignment=8) - .Title.data.[39] 239: 77386000..77839120 (len=453120, alignment=8) - .Title.data.[40] 240: 77839120..78292416 (len=453296, alignment=8) - .Title.data.[41] 241: 78292416..78761032 (len=468616, alignment=8) - .Title.data.[42] 242: 78761032..79223408 (len=462376, alignment=8) - .Title.data.[43] 243: 79223408..79683720 (len=460312, alignment=8) - .Title.data.[44] 244: 79683720..79696452 (len=12732, alignment=8) - .Title.data.[45] 245: 79696456..79696644 (len=188, alignment=8) - .GoodEvent.data 246: 79696648..81948016 (len=2251368, alignment=8) - .EventTime.data 247: 81948016..81948404 (len=388, alignment=8) - .EventDate.data 248: 81948408..82323856 (len=375448, alignment=8) - .CounterID.data 249: 82323856..83004996 (len=681140, alignment=8) - .ClientIP.data 250: 83005000..85010244 (len=2005244, alignment=8) - .RegionID.data 251: 85010248..86023652 (len=1013404, alignment=8) - .UserID.data 252: 86023656..86023776 (len=120, alignment=8) - .CounterClass.data 253: 86023776..87024164 (len=1000388, alignment=8) - .OS.data 254: 87024168..88024508 (len=1000340, alignment=8) - .UserAgent.data 255: 88024512..88353776 (len=329264, alignment=8) - .URL.data.[88] 256: 88353776..88514568 (len=160792, alignment=8) - .URL.data.[89] 257: 88514568..88815400 (len=300832, alignment=8) - .URL.data.[90] 258: 88815400..88987256 (len=171856, alignment=8) - .URL.data.[91] 259: 88987256..89268328 (len=281072, alignment=8) - .URL.data.[92] 260: 89268328..89619688 (len=351360, alignment=8) - .URL.data.[93] 261: 89619688..89919752 (len=300064, alignment=8) - .URL.data.[94] 262: 89919752..90077792 (len=158040, alignment=8) - .URL.data.[95] 263: 90077792..90335800 (len=258008, alignment=8) - .URL.data.[96] 264: 90335800..90601984 (len=266184, alignment=8) - .URL.data.[97] 265: 90601984..90866704 (len=264720, alignment=8) - .URL.data.[98] 266: 90866704..91123936 (len=257232, alignment=8) - .URL.data.[99] 267: 91123936..91380360 (len=256424, alignment=8) - .URL.data.[100] 268: 91380360..91641736 (len=261376, alignment=8) - .URL.data.[101] 269: 91641736..91909880 (len=268144, alignment=8) - .URL.data.[102] 270: 91909880..91919932 (len=10052, alignment=8) - .URL.data.[103] 271: 91919936..92195240 (len=275304, alignment=8) - .Referer.data.[81] 272: 92195240..92495456 (len=300216, alignment=8) - .Referer.data.[82] 273: 92495456..92770248 (len=274792, alignment=8) - .Referer.data.[83] 274: 92770248..93049696 (len=279448, alignment=8) - .Referer.data.[84] 275: 93049696..93337784 (len=288088, alignment=8) - .Referer.data.[85] 276: 93337784..93617808 (len=280024, alignment=8) - .Referer.data.[86] 277: 93617808..93900672 (len=282864, alignment=8) - .Referer.data.[87] 278: 93900672..93908732 (len=8060, alignment=8) - .Referer.data.[88] 279: 93908736..94033912 (len=125176, alignment=8) - .IsRefresh.data 280: 94033912..96034124 (len=2000212, alignment=8) - .RefererCategoryID.data 281: 96034128..97287732 (len=1253604, alignment=8) - .RefererRegionID.data 282: 97287736..98295716 (len=1007980, alignment=8) - .URLCategoryID.data 283: 98295720..99296604 (len=1000884, alignment=8) - .URLRegionID.data 284: 99296608..100528196 (len=1231588, alignment=8) - .ResolutionWidth.data 285: 100528200..102528404 (len=2000204, alignment=8) - .ResolutionHeight.data 286: 102528408..102903840 (len=375432, alignment=8) - .ResolutionDepth.data 287: 102903840..103279272 (len=375432, alignment=8) - .FlashMajor.data 288: 103279272..103779768 (len=500496, alignment=8) - .FlashMinor.data 289: 103779768..104087896 (len=308128, alignment=8) - .FlashMinor2.data.[0] 290: 104087904..104564556 (len=476652, alignment=16) - .FlashMinor2.data.[1] 291: 104564560..104814928 (len=250368, alignment=8) - .NetMajor.data 292: 104814928..105160812 (len=345884, alignment=8) - .NetMinor.data 293: 105160816..106161164 (len=1000348, alignment=8) - .UserAgentMajor.data 294: 106161168..106464512 (len=303344, alignment=8) - .UserAgentMinor.data.[0] 295: 106464512..106716248 (len=251736, alignment=8) - .UserAgentMinor.data.[1] 296: 106716248..106719124 (len=2876, alignment=8) - .CookieEnable.data 297: 106719128..106722004 (len=2876, alignment=8) - .JavascriptEnable.data 298: 106722008..106812196 (len=90188, alignment=8) - .IsMobile.data 299: 106812200..106975412 (len=163212, alignment=8) - .MobilePhone.data 300: 106975424..107028580 (len=53156, alignment=16) - .MobilePhoneModel.data.[0] 301: 107028592..107075284 (len=46692, alignment=16) - .MobilePhoneModel.data.[1] 302: 107075288..107075764 (len=476, alignment=8) - .Params.data.[0] 303: 107075768..107076244 (len=476, alignment=8) - .Params.data.[1] 304: 107076248..109129300 (len=2053052, alignment=8) - .IPNetworkID.data 305: 109129304..109506412 (len=377108, alignment=8) - .TraficSourceID.data 306: 109506416..109865104 (len=358688, alignment=8) - .SearchEngineID.data 307: 109865104..110094328 (len=229224, alignment=8) - .SearchPhrase.data.[0] 308: 110094328..110307024 (len=212696, alignment=8) - .SearchPhrase.data.[1] 309: 110307024..110468016 (len=160992, alignment=8) - .SearchPhrase.data.[2] 310: 110468016..110620776 (len=152760, alignment=8) - .SearchPhrase.data.[3] 311: 110620776..110778248 (len=157472, alignment=8) - .SearchPhrase.data.[4] 312: 110778248..110978640 (len=200392, alignment=8) - .SearchPhrase.data.[5] 313: 110978640..111009800 (len=31160, alignment=8) - .SearchPhrase.data.[6] 314: 111009800..111053456 (len=43656, alignment=8) - .AdvEngineID.data 315: 111053456..111178632 (len=125176, alignment=8) - .IsArtifical.data 316: 111178632..113178836 (len=2000204, alignment=8) - .WindowClientWidth.data 317: 113178840..115179044 (len=2000204, alignment=8) - .WindowClientHeight.data 318: 115179048..116179428 (len=1000380, alignment=8) - .ClientTimeZone.data 319: 116179432..118423876 (len=2244444, alignment=8) - .ClientEventTime.data 320: 118423880..118743892 (len=320012, alignment=8) - .SilverlightVersion1.data 321: 118743896..118869072 (len=125176, alignment=8) - .SilverlightVersion2.data 322: 118869072..119869492 (len=1000420, alignment=8) - .SilverlightVersion3.data 323: 119869496..119870012 (len=516, alignment=8) - .SilverlightVersion4.data 324: 119870016..120001468 (len=131452, alignment=16) - .PageCharset.data.[0] 325: 120001472..120120892 (len=119420, alignment=16) - .PageCharset.data.[1] 326: 120120896..120124304 (len=3408, alignment=8) - .CodeVersion.data 327: 120124304..120204252 (len=79948, alignment=8) - .IsLink.data 328: 120204256..120209964 (len=5708, alignment=8) - .IsDownload.data 329: 120209968..120293620 (len=83652, alignment=8) - .IsNotBounce.data 330: 120293624..121263732 (len=970108, alignment=8) - .FUniqID.data 331: 121263736..121457304 (len=193568, alignment=8) - .OriginalURL.data.[28] 332: 121457304..121742040 (len=284736, alignment=8) - .OriginalURL.data.[29] 333: 121742040..122099528 (len=357488, alignment=8) - .OriginalURL.data.[30] 334: 122099528..122249352 (len=149824, alignment=8) - .OriginalURL.data.[31] 335: 122249352..122605880 (len=356528, alignment=8) - .OriginalURL.data.[32] 336: 122605880..122926808 (len=320928, alignment=8) - .OriginalURL.data.[33] 337: 122926808..123063552 (len=136744, alignment=8) - .OriginalURL.data.[34] 338: 123063552..123345424 (len=281872, alignment=8) - .OriginalURL.data.[35] 339: 123345424..123472296 (len=126872, alignment=8) - .OriginalURL.data.[36] 340: 123472296..123611152 (len=138856, alignment=8) - .OriginalURL.data.[37] 341: 123611152..123949104 (len=337952, alignment=8) - .OriginalURL.data.[38] 342: 123949104..124093792 (len=144688, alignment=8) - .OriginalURL.data.[39] 343: 124093792..124271952 (len=178160, alignment=8) - .OriginalURL.data.[40] 344: 124271952..124540264 (len=268312, alignment=8) - .OriginalURL.data.[41] 345: 124540264..124671056 (len=130792, alignment=8) - .OriginalURL.data.[42] 346: 124671056..124823672 (len=152616, alignment=8) - .OriginalURL.data.[43] 347: 124823672..125140960 (len=317288, alignment=8) - .OriginalURL.data.[44] 348: 125140960..125237728 (len=96768, alignment=8) - .OriginalURL.data.[45] 349: 125237728..125281368 (len=43640, alignment=8) - .OriginalURL.data.[46] 350: 125281368..129281580 (len=4000212, alignment=8) - .HID.data 351: 129281584..129281704 (len=120, alignment=8) - .IsOldCounter.data 352: 129281704..129281824 (len=120, alignment=8) - .IsEvent.data 353: 129281824..129281944 (len=120, alignment=8) - .IsParameter.data 354: 129281944..129441252 (len=159308, alignment=8) - .DontCountHits.data 355: 129441256..129441376 (len=120, alignment=8) - .WithHash.data 356: 129441376..129513836 (len=72460, alignment=8) - .HitColor.data.[0] 357: 129513840..129577160 (len=63320, alignment=8) - .HitColor.data.[1] 358: 129577160..131828528 (len=2251368, alignment=8) - .LocalEventTime.data 359: 131828528..132203960 (len=375432, alignment=8) - .Age.data 360: 132203960..132454192 (len=250232, alignment=8) - .Sex.data 361: 132454192..132704424 (len=250232, alignment=8) - .Income.data 362: 132704424..134704628 (len=2000204, alignment=8) - .Interests.data 363: 134704632..135705840 (len=1001208, alignment=8) - .Robotness.data 364: 135705840..137944588 (len=2238748, alignment=8) - .RemoteIP.data 365: 137944592..138055620 (len=111028, alignment=8) - .WindowName.data 366: 138055624..138055812 (len=188, alignment=8) - .OpenerName.data 367: 138055816..138156628 (len=100812, alignment=8) - .HistoryLength.data 368: 138156632..138255744 (len=99112, alignment=8) - .BrowserLanguage.data.[0] 369: 138255744..138346232 (len=90488, alignment=8) - .BrowserLanguage.data.[1] 370: 138346232..138482472 (len=136240, alignment=8) - .BrowserCountry.data.[0] 371: 138482472..138606416 (len=123944, alignment=8) - .BrowserCountry.data.[1] 372: 138606416..138606892 (len=476, alignment=8) - .SocialNetwork.data.[0] 373: 138606896..138607372 (len=476, alignment=8) - .SocialNetwork.data.[1] 374: 138607376..138607852 (len=476, alignment=8) - .SocialAction.data.[0] 375: 138607856..138608332 (len=476, alignment=8) - .SocialAction.data.[1] 376: 138608336..138608456 (len=120, alignment=8) - .HTTPError.data 377: 138608456..138774932 (len=166476, alignment=8) - .SendTiming.data 378: 138774936..138963680 (len=188744, alignment=8) - .DNSTiming.data 379: 138963680..140027860 (len=1064180, alignment=8) - .ConnectTiming.data 380: 140027864..141772092 (len=1744228, alignment=8) - .ResponseStartTiming.data 381: 141772096..143225092 (len=1452996, alignment=8) - .ResponseEndTiming.data 382: 143225096..144624420 (len=1399324, alignment=8) - .FetchTiming.data 383: 144624424..144626036 (len=1612, alignment=8) - .SocialSourceNetworkID.data 384: 144626048..144627156 (len=1108, alignment=16) - .SocialSourcePage.data.[0] 385: 144627168..144628164 (len=996, alignment=16) - .SocialSourcePage.data.[1] 386: 144628168..144628288 (len=120, alignment=8) - .ParamPrice.data 387: 144628288..144628764 (len=476, alignment=8) - .ParamOrderID.data.[0] 388: 144628768..144629244 (len=476, alignment=8) - .ParamOrderID.data.[1] 389: 144629248..144629548 (len=300, alignment=16) - .ParamCurrency.data.[0] 390: 144629552..144629852 (len=300, alignment=16) - .ParamCurrency.data.[1] 391: 144629856..144629976 (len=120, alignment=8) - .ParamCurrencyID.data 392: 144629984..144641876 (len=11892, alignment=16) - .OpenstatServiceName.data.[0] 393: 144641888..144653428 (len=11540, alignment=16) - .OpenstatServiceName.data.[1] 394: 144653440..144664244 (len=10804, alignment=16) - .OpenstatCampaignID.data.[0] 395: 144664256..144674212 (len=9956, alignment=16) - .OpenstatCampaignID.data.[1] 396: 144674224..144685588 (len=11364, alignment=16) - .OpenstatAdID.data.[0] 397: 144685600..144696292 (len=10692, alignment=16) - .OpenstatAdID.data.[1] 398: 144696304..144703700 (len=7396, alignment=16) - .OpenstatSourceID.data.[0] 399: 144703704..144710120 (len=6416, alignment=8) - .OpenstatSourceID.data.[1] 400: 144710128..144721620 (len=11492, alignment=16) - .UTMSource.data.[0] 401: 144721632..144732276 (len=10644, alignment=16) - .UTMSource.data.[1] 402: 144732288..144743556 (len=11268, alignment=16) - .UTMMedium.data.[0] 403: 144743568..144753828 (len=10260, alignment=16) - .UTMMedium.data.[1] 404: 144753832..144768624 (len=14792, alignment=8) - .UTMCampaign.data.[0] 405: 144768624..144782288 (len=13664, alignment=8) - .UTMCampaign.data.[1] 406: 144782288..144789636 (len=7348, alignment=16) - .UTMContent.data.[0] 407: 144789648..144796660 (len=7012, alignment=16) - .UTMContent.data.[1] 408: 144796664..144805288 (len=8624, alignment=8) - .UTMTerm.data.[0] 409: 144805296..144812324 (len=7028, alignment=16) - .UTMTerm.data.[1] 410: 144812336..144829444 (len=17108, alignment=16) - .FromTag.data.[0] 411: 144829456..144848292 (len=18836, alignment=16) - .FromTag.data.[1] 412: 144848296..144871924 (len=23628, alignment=8) - .HasGCLID.data 413: 144871928..152872140 (len=8000212, alignment=8) - .RefererHash.data 414: 152872144..160872356 (len=8000212, alignment=8) - .URLHash.data 415: 160872360..160873460 (len=1100, alignment=8) - .CLID.data 416: 160873464..160875900 (len=2436, alignment=8) - .WatchID.stats_table 417: 160875904..160877412 (len=1508, alignment=8) - .JavaEnable.stats_table 418: 160877424..160895188 (len=17764, alignment=16) - .Title.stats_table 419: 160895192..160895876 (len=684, alignment=8) - .GoodEvent.stats_table 420: 160895880..160898092 (len=2212, alignment=8) - .EventTime.stats_table 421: 160898096..160899284 (len=1188, alignment=8) - .EventDate.stats_table 422: 160899288..160901812 (len=2524, alignment=8) - .CounterID.stats_table 423: 160901816..160904372 (len=2556, alignment=8) - .ClientIP.stats_table 424: 160904376..160906556 (len=2180, alignment=8) - .RegionID.stats_table 425: 160906560..160908996 (len=2436, alignment=8) - .UserID.stats_table 426: 160909000..160909384 (len=384, alignment=8) - .CounterClass.stats_table 427: 160909384..160911324 (len=1940, alignment=8) - .OS.stats_table 428: 160911328..160913396 (len=2068, alignment=8) - .UserAgent.stats_table 429: 160913400..160921556 (len=8156, alignment=8) - .URL.stats_table 430: 160921560..160927500 (len=5940, alignment=8) - .Referer.stats_table 431: 160927504..160929012 (len=1508, alignment=8) - .IsRefresh.stats_table 432: 160929016..160930764 (len=1748, alignment=8) - .RefererCategoryID.stats_table 433: 160930768..160932764 (len=1996, alignment=8) - .RefererRegionID.stats_table 434: 160932768..160934716 (len=1948, alignment=8) - .URLCategoryID.stats_table 435: 160934720..160936980 (len=2260, alignment=8) - .URLRegionID.stats_table 436: 160936984..160938900 (len=1916, alignment=8) - .ResolutionWidth.stats_table 437: 160938904..160940820 (len=1916, alignment=8) - .ResolutionHeight.stats_table 438: 160940824..160942516 (len=1692, alignment=8) - .ResolutionDepth.stats_table 439: 160942520..160944036 (len=1516, alignment=8) - .FlashMajor.stats_table 440: 160944040..160945556 (len=1516, alignment=8) - .FlashMinor.stats_table 441: 160945568..160946556 (len=988, alignment=16) - .FlashMinor2.stats_table 442: 160946560..160948164 (len=1604, alignment=8) - .NetMajor.stats_table 443: 160948168..160949676 (len=1508, alignment=8) - .NetMinor.stats_table 444: 160949680..160951748 (len=2068, alignment=8) - .UserAgentMajor.stats_table 445: 160951752..160953116 (len=1364, alignment=8) - .UserAgentMinor.stats_table 446: 160953120..160954748 (len=1628, alignment=8) - .CookieEnable.stats_table 447: 160954752..160956380 (len=1628, alignment=8) - .JavascriptEnable.stats_table 448: 160956384..160957980 (len=1596, alignment=8) - .IsMobile.stats_table 449: 160957984..160959724 (len=1740, alignment=8) - .MobilePhone.stats_table 450: 160959728..160960748 (len=1020, alignment=16) - .MobilePhoneModel.stats_table 451: 160960752..160961732 (len=980, alignment=8) - .Params.stats_table 452: 160961736..160964060 (len=2324, alignment=8) - .IPNetworkID.stats_table 453: 160964064..160965884 (len=1820, alignment=8) - .TraficSourceID.stats_table 454: 160965888..160967628 (len=1740, alignment=8) - .SearchEngineID.stats_table 455: 160967632..160973104 (len=5472, alignment=8) - .SearchPhrase.stats_table 456: 160973104..160974844 (len=1740, alignment=8) - .AdvEngineID.stats_table 457: 160974848..160976356 (len=1508, alignment=8) - .IsArtifical.stats_table 458: 160976360..160978300 (len=1940, alignment=8) - .WindowClientWidth.stats_table 459: 160978304..160980236 (len=1932, alignment=8) - .WindowClientHeight.stats_table 460: 160980240..160982172 (len=1932, alignment=8) - .ClientTimeZone.stats_table 461: 160982176..160984800 (len=2624, alignment=8) - .ClientEventTime.stats_table 462: 160984800..160986420 (len=1620, alignment=8) - .SilverlightVersion1.stats_table 463: 160986424..160987932 (len=1508, alignment=8) - .SilverlightVersion2.stats_table 464: 160987936..160989580 (len=1644, alignment=8) - .SilverlightVersion3.stats_table 465: 160989584..160990440 (len=856, alignment=8) - .SilverlightVersion4.stats_table 466: 160990448..160991484 (len=1036, alignment=16) - .PageCharset.stats_table 467: 160991488..160993700 (len=2212, alignment=8) - .CodeVersion.stats_table 468: 160993704..160995396 (len=1692, alignment=8) - .IsLink.stats_table 469: 160995400..160996468 (len=1068, alignment=8) - .IsDownload.stats_table 470: 160996472..160997468 (len=996, alignment=8) - .IsNotBounce.stats_table 471: 160997472..160998844 (len=1372, alignment=8) - .FUniqID.stats_table 472: 160998848..161013480 (len=14632, alignment=8) - .OriginalURL.stats_table 473: 161013480..161015700 (len=2220, alignment=8) - .HID.stats_table 474: 161015704..161016088 (len=384, alignment=8) - .IsOldCounter.stats_table 475: 161016088..161016472 (len=384, alignment=8) - .IsEvent.stats_table 476: 161016472..161016856 (len=384, alignment=8) - .IsParameter.stats_table 477: 161016856..161018452 (len=1596, alignment=8) - .DontCountHits.stats_table 478: 161018456..161018840 (len=384, alignment=8) - .WithHash.stats_table 479: 161018840..161019996 (len=1156, alignment=8) - .HitColor.stats_table 480: 161020000..161022212 (len=2212, alignment=8) - .LocalEventTime.stats_table 481: 161022216..161023732 (len=1516, alignment=8) - .Age.stats_table 482: 161023736..161025244 (len=1508, alignment=8) - .Sex.stats_table 483: 161025248..161026756 (len=1508, alignment=8) - .Income.stats_table 484: 161026760..161028508 (len=1748, alignment=8) - .Interests.stats_table 485: 161028512..161030260 (len=1748, alignment=8) - .Robotness.stats_table 486: 161030264..161032820 (len=2556, alignment=8) - .RemoteIP.stats_table 487: 161032824..161034884 (len=2060, alignment=8) - .WindowName.stats_table 488: 161034888..161035572 (len=684, alignment=8) - .OpenerName.stats_table 489: 161035576..161037396 (len=1820, alignment=8) - .HistoryLength.stats_table 490: 161037400..161038804 (len=1404, alignment=8) - .BrowserLanguage.stats_table 491: 161038816..161039980 (len=1164, alignment=16) - .BrowserCountry.stats_table 492: 161039984..161040964 (len=980, alignment=8) - .SocialNetwork.stats_table 493: 161040968..161041948 (len=980, alignment=8) - .SocialAction.stats_table 494: 161041952..161042336 (len=384, alignment=8) - .HTTPError.stats_table 495: 161042336..161044332 (len=1996, alignment=8) - .SendTiming.stats_table 496: 161044336..161046332 (len=1996, alignment=8) - .DNSTiming.stats_table 497: 161046336..161048332 (len=1996, alignment=8) - .ConnectTiming.stats_table 498: 161048336..161050332 (len=1996, alignment=8) - .ResponseStartTiming.stats_table 499: 161050336..161052332 (len=1996, alignment=8) - .ResponseEndTiming.stats_table 500: 161052336..161054332 (len=1996, alignment=8) - .FetchTiming.stats_table 501: 161054336..161055484 (len=1148, alignment=8) - .SocialSourceNetworkID.stats_table 502: 161055488..161057100 (len=1612, alignment=16) - .SocialSourcePage.stats_table 503: 161057104..161057488 (len=384, alignment=8) - .ParamPrice.stats_table 504: 161057488..161058468 (len=980, alignment=8) - .ParamOrderID.stats_table 505: 161058480..161059124 (len=644, alignment=16) - .ParamCurrency.stats_table 506: 161059128..161059512 (len=384, alignment=8) - .ParamCurrencyID.stats_table 507: 161059520..161060572 (len=1052, alignment=16) - .OpenstatServiceName.stats_table 508: 161060576..161061740 (len=1164, alignment=16) - .OpenstatCampaignID.stats_table 509: 161061744..161063260 (len=1516, alignment=16) - .OpenstatAdID.stats_table 510: 161063264..161064380 (len=1116, alignment=16) - .OpenstatSourceID.stats_table 511: 161064384..161065852 (len=1468, alignment=16) - .UTMSource.stats_table 512: 161065856..161067004 (len=1148, alignment=8) - .UTMMedium.stats_table 513: 161067008..161069180 (len=2172, alignment=16) - .UTMCampaign.stats_table 514: 161069184..161071084 (len=1900, alignment=16) - .UTMContent.stats_table 515: 161071088..161073372 (len=2284, alignment=16) - .UTMTerm.stats_table 516: 161073376..161074380 (len=1004, alignment=16) - .FromTag.stats_table 517: 161074384..161075940 (len=1556, alignment=8) - .HasGCLID.stats_table 518: 161075944..161078380 (len=2436, alignment=8) - .RefererHash.stats_table 519: 161078384..161080820 (len=2436, alignment=8) - .URLHash.stats_table 520: 161080824..161082084 (len=1260, alignment=8) - .CLID.stats_table ```
1 parent f69f00c commit 3651278

File tree

5 files changed

+38
-60
lines changed

5 files changed

+38
-60
lines changed

‎vortex-file/src/strategy.rs‎

Lines changed: 21 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
//! This module defines the default layout strategy for a Vortex file.
22
3+
use std::collections::VecDeque;
34
use std::sync::Arc;
45

56
use vortex_array::arcref::ArcRef;
@@ -9,7 +10,7 @@ use vortex_array::{Array, ArrayContext, ArrayRef};
910
use vortex_btrblocks::BtrBlocksCompressor;
1011
use vortex_dtype::DType;
1112
use vortex_error::VortexResult;
12-
use vortex_layout::layouts::chunked::writer::{ChunkedLayoutOptions, ChunkedLayoutWriter};
13+
use vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy;
1314
use vortex_layout::layouts::flat::FlatLayout;
1415
use vortex_layout::layouts::stats::writer::{StatsLayoutOptions, StatsLayoutWriter};
1516
use vortex_layout::layouts::struct_::writer::StructLayoutWriter;
@@ -32,27 +33,18 @@ impl LayoutStrategy for VortexLayoutStrategy {
3233
);
3334
}
3435

35-
// Leaf arrays are written as flat arrays, above which, we buffer up to 16MB to try to keep
36-
// chunks for the same column next to each other, within some reasonable write-time memory
37-
// buffering limit.
38-
let writer: ArcRef<dyn LayoutStrategy> = ArcRef::new_arc(Arc::new(BufferedStrategy {
39-
child: ArcRef::new_ref(&FlatLayout),
40-
// Buffer 4MB of compressed data per column before writing the chunks consecutively.
36+
// We buffer arrays per column, before flushing them into a chunked layout.
37+
// This helps to keep consecutive chunks of a column adjacent for more efficient reads.
38+
let strategy: ArcRef<dyn LayoutStrategy> = ArcRef::new_arc(Arc::new(BufferedStrategy {
39+
child: ArcRef::new_arc(Arc::new(ChunkedLayoutStrategy::default()) as _),
4140
// TODO(ngates): this should really be amortized by the number of fields? Maybe the
4241
// strategy could keep track of how many writers were created?
43-
buffer_size: 4 << 20, // 4 MB
42+
buffer_size: 2 << 20, // 2MB
4443
}) as _);
4544

4645
// Compress each chunk with btrblocks.
4746
let writer = BtrBlocksCompressedWriter {
48-
child: ChunkedLayoutWriter::new(
49-
ctx.clone(),
50-
&DType::Null,
51-
ChunkedLayoutOptions {
52-
chunk_strategy: writer,
53-
},
54-
)
55-
.boxed(),
47+
child: strategy.new_writer(ctx, dtype)?,
5648
}
5749
.boxed();
5850

@@ -146,7 +138,7 @@ impl LayoutStrategy for BufferedStrategy {
146138
fn new_writer(&self, ctx: &ArrayContext, dtype: &DType) -> VortexResult<Box<dyn LayoutWriter>> {
147139
let child = self.child.new_writer(ctx, dtype)?;
148140
Ok(BufferedWriter {
149-
chunks: Vec::new(),
141+
chunks: Default::default(),
150142
nbytes: 0,
151143
buffer_size: self.buffer_size,
152144
child,
@@ -156,7 +148,7 @@ impl LayoutStrategy for BufferedStrategy {
156148
}
157149

158150
struct BufferedWriter {
159-
chunks: Vec<ArrayRef>,
151+
chunks: VecDeque<ArrayRef>,
160152
nbytes: u64,
161153
buffer_size: u64,
162154
child: Box<dyn LayoutWriter>,
@@ -169,12 +161,18 @@ impl LayoutWriter for BufferedWriter {
169161
chunk: ArrayRef,
170162
) -> VortexResult<()> {
171163
self.nbytes += chunk.nbytes() as u64;
172-
self.chunks.push(chunk);
173-
if self.nbytes >= self.buffer_size {
174-
for chunk in self.chunks.drain(..) {
175-
self.child.push_chunk(segment_writer, chunk)?;
164+
self.chunks.push_back(chunk);
165+
// Wait until we're at 2x the buffer size before flushing 1x the buffer size
166+
// This avoids small tail stragglers being flushed at the end of the file.
167+
if self.nbytes >= 2 * self.buffer_size {
168+
while self.nbytes > self.buffer_size {
169+
if let Some(chunk) = self.chunks.pop_front() {
170+
self.nbytes -= chunk.nbytes() as u64;
171+
self.child.push_chunk(segment_writer, chunk)?;
172+
} else {
173+
break;
174+
}
176175
}
177-
self.nbytes = 0;
178176
}
179177
Ok(())
180178
}

‎vortex-layout/src/layouts/chunked/eval_expr.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ mod test {
9292
let mut segments = TestSegments::default();
9393
let layout = ChunkedLayoutWriter::new(
9494
ctx.clone(),
95-
&DType::Primitive(PType::I32, NonNullable),
95+
DType::Primitive(PType::I32, NonNullable),
9696
Default::default(),
9797
)
9898
.push_all(

‎vortex-layout/src/layouts/chunked/writer.rs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,45 +3,52 @@ use vortex_array::{ArrayContext, ArrayRef};
33
use vortex_dtype::DType;
44
use vortex_error::{VortexExpect, VortexResult};
55

6-
use crate::LayoutVTableRef;
76
use crate::data::Layout;
87
use crate::layouts::chunked::ChunkedLayout;
98
use crate::layouts::flat::FlatLayout;
109
use crate::segments::SegmentWriter;
1110
use crate::strategy::LayoutStrategy;
1211
use crate::writer::LayoutWriter;
12+
use crate::{LayoutVTableRef, LayoutWriterExt};
1313

14-
pub struct ChunkedLayoutOptions {
14+
#[derive(Clone)]
15+
pub struct ChunkedLayoutStrategy {
1516
/// The layout strategy for each chunk.
1617
pub chunk_strategy: ArcRef<dyn LayoutStrategy>,
1718
}
1819

19-
impl Default for ChunkedLayoutOptions {
20+
impl Default for ChunkedLayoutStrategy {
2021
fn default() -> Self {
2122
Self {
2223
chunk_strategy: ArcRef::new_ref(&FlatLayout),
2324
}
2425
}
2526
}
2627

28+
impl LayoutStrategy for ChunkedLayoutStrategy {
29+
fn new_writer(&self, ctx: &ArrayContext, dtype: &DType) -> VortexResult<Box<dyn LayoutWriter>> {
30+
Ok(ChunkedLayoutWriter::new(ctx.clone(), dtype.clone(), self.clone()).boxed())
31+
}
32+
}
33+
2734
/// A basic implementation of a chunked layout writer that writes each batch into its own chunk.
2835
///
2936
/// TODO(ngates): introduce more sophisticated layout writers with different chunking strategies.
3037
pub struct ChunkedLayoutWriter {
3138
ctx: ArrayContext,
32-
options: ChunkedLayoutOptions,
39+
options: ChunkedLayoutStrategy,
3340
chunks: Vec<Box<dyn LayoutWriter>>,
3441
dtype: DType,
3542
row_count: u64,
3643
}
3744

3845
impl ChunkedLayoutWriter {
39-
pub fn new(ctx: ArrayContext, dtype: &DType, options: ChunkedLayoutOptions) -> Self {
46+
pub fn new(ctx: ArrayContext, dtype: DType, options: ChunkedLayoutStrategy) -> Self {
4047
Self {
4148
ctx,
4249
options,
43-
chunks: Vec::new(),
44-
dtype: dtype.clone(),
50+
chunks: vec![],
51+
dtype,
4552
row_count: 0,
4653
}
4754
}
@@ -69,6 +76,7 @@ impl LayoutWriter for ChunkedLayoutWriter {
6976
}
7077

7178
fn flush(&mut self, _segment_writer: &mut dyn SegmentWriter) -> VortexResult<()> {
79+
// We flush each chunk as we write it, so there's nothing to do here.
7280
Ok(())
7381
}
7482

‎vortex-layout/src/layouts/stats/eval_expr.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ mod test {
9797
&DType::Primitive(PType::I32, NonNullable),
9898
ChunkedLayoutWriter::new(
9999
ctx.clone(),
100-
&DType::Primitive(PType::I32, NonNullable),
100+
DType::Primitive(PType::I32, NonNullable),
101101
Default::default(),
102102
)
103103
.boxed(),

‎vortex-layout/src/strategy.rs‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,9 @@
44
//! all while remaining independent of the read code.
55
66
use vortex_array::ArrayContext;
7-
use vortex_array::arcref::ArcRef;
87
use vortex_dtype::DType;
98
use vortex_error::VortexResult;
109

11-
use crate::layouts::chunked::writer::{ChunkedLayoutOptions, ChunkedLayoutWriter};
1210
use crate::layouts::flat::FlatLayout;
1311
use crate::layouts::flat::writer::FlatLayoutWriter;
1412
use crate::layouts::struct_::writer::StructLayoutWriter;
@@ -38,29 +36,3 @@ impl LayoutStrategy for StructStrategy {
3836
}
3937
}
4038
}
41-
42-
/// A layout strategy that preserves each chunk as-given.
43-
pub struct ChunkedStrategy {
44-
pub chunk_strategy: ArcRef<dyn LayoutStrategy>,
45-
}
46-
47-
impl Default for ChunkedStrategy {
48-
fn default() -> Self {
49-
Self {
50-
chunk_strategy: ArcRef::new_ref(&StructStrategy),
51-
}
52-
}
53-
}
54-
55-
impl LayoutStrategy for ChunkedStrategy {
56-
fn new_writer(&self, ctx: &ArrayContext, dtype: &DType) -> VortexResult<Box<dyn LayoutWriter>> {
57-
Ok(ChunkedLayoutWriter::new(
58-
ctx.clone(),
59-
dtype,
60-
ChunkedLayoutOptions {
61-
chunk_strategy: self.chunk_strategy.clone(),
62-
},
63-
)
64-
.boxed())
65-
}
66-
}

0 commit comments

Comments
 (0)