[AVX-512] Look for vector bit rotates on vectors larger than 16 bytes

[Validark]

Zig Godbolt
LLVM Godbolt

define dso_local <2 x i64> @foo(<2 x i64> %0, <2 x i64> %1) local_unnamed_addr {
Entry:
  %2 = trunc <2 x i64> %1 to <2 x i6>
  %3 = sub <2 x i6> zeroinitializer, %2
  %4 = zext <2 x i6> %3 to <2 x i64>
  %5 = shl <2 x i64> %0, %4
  %6 = and <2 x i64> %1, splat (i64 63)
  %7 = lshr <2 x i64> %0, %6
  %8 = or <2 x i64> %5, %7
  ret <2 x i64> %8
}

define dso_local <4 x i64> @bar(<4 x i64> %0, <4 x i64> %1) local_unnamed_addr {
Entry:
  %2 = trunc <4 x i64> %1 to <4 x i6>
  %3 = sub <4 x i6> zeroinitializer, %2
  %4 = zext <4 x i6> %3 to <4 x i64>
  %5 = shl <4 x i64> %0, %4
  %6 = and <4 x i64> %1, splat (i64 63)
  %7 = lshr <4 x i64> %0, %6
  %8 = or <4 x i64> %5, %7
  ret <4 x i64> %8
}

define dso_local <8 x i64> @baz(<8 x i64> %0, <8 x i64> %1) local_unnamed_addr {
Entry:
  %2 = trunc <8 x i64> %1 to <8 x i6>
  %3 = sub <8 x i6> zeroinitializer, %2
  %4 = zext <8 x i6> %3 to <8 x i64>
  %5 = shl <8 x i64> %0, %4
  %6 = and <8 x i64> %1, splat (i64 63)
  %7 = lshr <8 x i64> %0, %6
  %8 = or <8 x i64> %5, %7
  ret <8 x i64> %8
}

Emit:

foo:
        vprorvq xmm0, xmm0, xmm1
        ret

.LCPI1_0:
        .quad   63
bar:
        vpbroadcastq    ymm4, qword ptr [rip + .LCPI1_0]
        vpmovqd xmm2, ymm1
        vpxor   xmm3, xmm3, xmm3
        vpsubd  xmm2, xmm3, xmm2
        vpmovzxdq       ymm2, xmm2
        vpand   ymm2, ymm2, ymm4
        vpand   ymm1, ymm1, ymm4
        vpsllvq ymm2, ymm0, ymm2
        vpsrlvq ymm0, ymm0, ymm1
        vpor    ymm0, ymm2, ymm0
        ret

.LCPI2_0:
        .quad   63
baz:
        vpmovqw xmm2, zmm1
        vpbroadcastq    zmm4, qword ptr [rip + .LCPI2_0]
        vpxor   xmm3, xmm3, xmm3
        vpsubw  xmm2, xmm3, xmm2
        vpmovzxwq       zmm2, xmm2
        vpandq  zmm1, zmm1, zmm4
        vpandq  zmm2, zmm2, zmm4
        vpsllvq zmm2, zmm0, zmm2
        vpsrlvq zmm0, zmm0, zmm1
        vporq   zmm0, zmm2, zmm0
        ret

[llvmbot]

@llvm/issue-subscribers-backend-x86

Author: Niles Salter (Validark)

[Zig Godbolt](https://zig.godbo.lt/#g:!((g:!((g:!((h:codeEditor,i:(filename:'1',fontScale:14,fontUsePx:'0',j:1,lang:zig,selection:(endColumn:2,endLineNumber:11,positionColumn:2,positionLineNumber:11,selectionStartColumn:2,selectionStartLineNumber:11,startColumn:2,startLineNumber:11),source:'export+fn+foo(x:+@Vector(2,+u64),+i:+@Vector(2,+u64))+@Vector(2,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D%0A%0Aexport+fn+bar(x:+@Vector(4,+u64),+i:+@Vector(4,+u64))+@Vector(4,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D%0A%0Aexport+fn+baz(x:+@Vector(8,+u64),+i:+@Vector(8,+u64))+@Vector(8,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D'),l:'5',n:'0',o:'Zig+source+%231',t:'0')),k:51.27826974307506,l:'4',m:100,n:'0',o:'',s:0,t:'0'),(g:!((h:compiler,i:(compiler:ztrunk,filters:(b:'0',binary:'1',binaryObject:'1',commentOnly:'0',debugCalls:'1',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1',verboseDemangling:'0'),flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:2,lang:zig,libs:!(),options:'-O+ReleaseFast+-target+x86_64-linux+-mcpu%3Dznver5+-fomit-frame-pointer',overrides:!(),selection:(endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1),source:1),l:'5',n:'0',o:'+zig+trunk+(Editor+%231)',t:'0')),header:(),k:48.72173025692494,l:'4',m:100.00000000000001,n:'0',o:'',s:0,t:'0')),l:'2',m:100,n:'0',o:'',t:'0')),version:4) [LLVM Godbolt](https://llvm.godbo.lt/#z:OYLghAFBqd5QCxAYwPYBMCmBRdBLAF1QCcAaPECAMzwBtMA7AQwFtMQByARg9KtQYEAysib0QXACx8BBAKoBnTAAUAHpwAMvAFYTStJg1C1aANxakl9ZATwDKjdAGFUtAK4sGexwBk8DTAA5DwAjTGIQAGYANlIAB1QFQjsGF3dPPQSk2wE/AOCWMIiYy0xrHIYhAiZiAjSPLy5S8pSqmoI8oNDwqNiFatr6jKb%2B9s6Cot6ASktUN2Jkdg4sGgCAanQFVAB9WlRRWjWAUkinACY11TW8aMkT7DWAAX5UCBPzy%2Bvb%2B%2BOzgFYNKRjqcLlcbndIg8jv8uFM1nsDts3AxmGx0NsmOh0MRjgB2ABCRw0AEFsIJiABPEBE4lrX5/C4nAAiawIxGRyGBHzB30h9K4rNQXNBX3uNLp0L%2BkWBLIUbhCws%2BNx%2BAC9wqh/MkxHg1WR6WdxfTJDK1mrVARFTyfpLpURLV8IdhDZK/iaFAhDu8ReDrf9AUbnf9oibDOh7T6%2BZKmmsFHEDBaIOC1tFIlNA39cSbaO6cV6lbyoX6gZLoumABwmkjhgv0v7F/64w3ETAWvM8x30ss0o64pndkkrfyYDZbXb7MRc43tn6PEI1N6nKcO30A4uL/MdqNwhFiJEo1iYdGY7F4wkkslsqnpxmRFlsjmTjcrgV295L5WQ9PS5kx%2BWPq18rqGoMFqtA6uE9YMumxo/marbrgBhZSoK/7Lp%2BJISv8ro/u6noIWhSH%2BpKdwYfSwY/qGqERkh0axvGayJrcyapummY/tmCC5vh1H0kRQblpWXFOO%2BNYupBjakc28HCU%2Bkb/F2JI9n2ikDpgqzDpsOw7nhTgVtOfKzkwKoLrpsmEWupn6TR27jrQe6ooeGJYrmBI0helLUqRko3ne7IMJy7x6QR/IoYF%2BZil5/zfrev4KmFiGmuqmq2NquqQQakV/DBMVwVyQUfkhtpCvFBHpthMW4XlZm8ZBJG0mRIYMGGJU8VGQJ0UwCZJimaaZWxMUcUJ%2BWiUWZECT%2BVYtSNfx1vSEn1VJVVWZ23a9jSHAzLQnB/LwXgcFopCoJwPg%2BAAagAsmsACSABKMZzAsw7QpEPCkAQmgbTMADWIBZQAdJIuIAJwaEDuJZYDINg9IW0cJIu0fYdnC8AoICAu9%2B0baQcCwEgaAsHEdAQRQED44T9ARCYNh%2BV9fB0AQ4SoxAISIyE/g1FS3C8GzzCUgA8iE2iYDYnCvfjbCCHzDC0JzB1YCEbjAE4YjZqLvBYCwhjAOImOkPgzY2HgpiYKjuuYKowtuAzavkIIZSI2BITEBzLhYIjbJ4CwNvG8QISJJgTKYJrRhgUYH0zFQBjAAop14JgADufNxIwNv8IIIhiOwUgyIIigqOouu6E0BhhyY5j6HgISo5AMyoHEFSmwAtHz0qNywyBxG4zIqgwPs7YdPvEHgWDVxAMxWMLFQOE1QyNKQvhDhMPRNFkyQCLPmSJGvDDjN0EQjGUk%2BtAMdSuA0egT4bAhtLUu%2BFMvlgnxvIwn3fkywrM8yLBIm3bQjutHQ4PCWgnIID3gYF9OEEBcCEEmmcF6UxeAYy0FMb6IBJB/D%2BgyUGGgzgYOwSDM4sRYbw1IF7LgGhAR7QOoAlGaM3rh1/hwM4/8aHIwYZjVBpAfZJHsJIIAA%3D%3D)

define dso_local &lt;2 x i64&gt; @<!-- -->foo(&lt;2 x i64&gt; %0, &lt;2 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;2 x i64&gt; %1 to &lt;2 x i6&gt;
  %3 = sub &lt;2 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;2 x i6&gt; %3 to &lt;2 x i64&gt;
  %5 = shl &lt;2 x i64&gt; %0, %4
  %6 = and &lt;2 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;2 x i64&gt; %0, %6
  %8 = or &lt;2 x i64&gt; %5, %7
  ret &lt;2 x i64&gt; %8
}

define dso_local &lt;4 x i64&gt; @<!-- -->bar(&lt;4 x i64&gt; %0, &lt;4 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;4 x i64&gt; %1 to &lt;4 x i6&gt;
  %3 = sub &lt;4 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;4 x i6&gt; %3 to &lt;4 x i64&gt;
  %5 = shl &lt;4 x i64&gt; %0, %4
  %6 = and &lt;4 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;4 x i64&gt; %0, %6
  %8 = or &lt;4 x i64&gt; %5, %7
  ret &lt;4 x i64&gt; %8
}

define dso_local &lt;8 x i64&gt; @<!-- -->baz(&lt;8 x i64&gt; %0, &lt;8 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;8 x i64&gt; %1 to &lt;8 x i6&gt;
  %3 = sub &lt;8 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;8 x i6&gt; %3 to &lt;8 x i64&gt;
  %5 = shl &lt;8 x i64&gt; %0, %4
  %6 = and &lt;8 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;8 x i64&gt; %0, %6
  %8 = or &lt;8 x i64&gt; %5, %7
  ret &lt;8 x i64&gt; %8
}

Emit:

foo:
        vprorvq xmm0, xmm0, xmm1
        ret

.LCPI1_0:
        .quad   63
bar:
        vpbroadcastq    ymm4, qword ptr [rip + .LCPI1_0]
        vpmovqd xmm2, ymm1
        vpxor   xmm3, xmm3, xmm3
        vpsubd  xmm2, xmm3, xmm2
        vpmovzxdq       ymm2, xmm2
        vpand   ymm2, ymm2, ymm4
        vpand   ymm1, ymm1, ymm4
        vpsllvq ymm2, ymm0, ymm2
        vpsrlvq ymm0, ymm0, ymm1
        vpor    ymm0, ymm2, ymm0
        ret

.LCPI2_0:
        .quad   63
baz:
        vpmovqw xmm2, zmm1
        vpbroadcastq    zmm4, qword ptr [rip + .LCPI2_0]
        vpxor   xmm3, xmm3, xmm3
        vpsubw  xmm2, xmm3, xmm2
        vpmovzxwq       zmm2, xmm2
        vpandq  zmm1, zmm1, zmm4
        vpandq  zmm2, zmm2, zmm4
        vpsllvq zmm2, zmm0, zmm2
        vpsrlvq zmm0, zmm0, zmm1
        vporq   zmm0, zmm2, zmm0
        ret

[topperc]

Looks like the trunc and zext around the negate are breaking it. This works

define dso_local <4 x i64> @bar(<4 x i64> %0, <4 x i64> %1) { 
Entry:
  %3 = sub <4 x i64> zeroinitializer, %1
  %4 = and <4 x i64> %3, splat (i64 63)
  %5 = shl <4 x i64> %0, %4
  %6 = and <4 x i64> %1, splat (i64 63)
  %7 = lshr <4 x i64> %0, %6
  %8 = or <4 x i64> %5, %7
  ret <4 x i64> %8
}

define dso_local <8 x i64> @baz(<8 x i64> %0, <8 x i64> %1) { 
Entry:
  %3 = sub <8 x i64> zeroinitializer, %1
  %4 = and <8 x i64> %3, splat (i64 63)
  %5 = shl <8 x i64> %0, %4
  %6 = and <8 x i64> %1, splat (i64 63)
  %7 = lshr <8 x i64> %0, %6
  %8 = or <8 x i64> %5, %7
  ret <8 x i64> %8
}

[RKSimon]

We could handle this in InstCombine with:

----------------------------------------
define i64 @src(i64 %a0) {
  %trunc = trunc i64 %a0 to i6
  %neg = sub i6 0, %trunc
  %zext = zext i6 %neg to i64
  ret i64 %zext
}
=>
define i64 @tgt(i64 %a0) {
  %neg = sub i64 0, %a0
  %mask = and i64 %neg, 63
  ret i64 %mask
}
Transformation seems to be correct!

There might be a more general sub pattern but the negate might be a good starting point.

[llvmbot]

Hi!

This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are:

1. Check that no other contributor is working on this issue. If someone is assigned to the issue or claimed to be working on it, ping the person. After one week without a response, the assignee may be changed.
2. Leave a comment indicating that you are working on the issue, or just create a pull request after following the steps below. Mention this issue in the description of the pull request.
3. Fix the issue locally.
4. Run the test suite locally. Remember that the subdirectories under test/ create fine-grained testing targets, so you can e.g. use make check-clang-ast to only run Clang's AST tests.
5. Create a Git commit.
6. Run git clang-format HEAD~1 to format your changes.
7. Open a pull request to the upstream repository on GitHub. Detailed instructions can be found in GitHub's documentation. Mention this issue in the description of the pull request.

If you have any further questions about this issue, don't hesitate to ask via a comment in the thread below.

[llvmbot]

@llvm/issue-subscribers-good-first-issue

Author: Niles Salter (Validark)

[Zig Godbolt](https://zig.godbo.lt/#g:!((g:!((g:!((h:codeEditor,i:(filename:'1',fontScale:14,fontUsePx:'0',j:1,lang:zig,selection:(endColumn:2,endLineNumber:11,positionColumn:2,positionLineNumber:11,selectionStartColumn:2,selectionStartLineNumber:11,startColumn:2,startLineNumber:11),source:'export+fn+foo(x:+@Vector(2,+u64),+i:+@Vector(2,+u64))+@Vector(2,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D%0A%0Aexport+fn+bar(x:+@Vector(4,+u64),+i:+@Vector(4,+u64))+@Vector(4,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D%0A%0Aexport+fn+baz(x:+@Vector(8,+u64),+i:+@Vector(8,+u64))+@Vector(8,+u64)+%7B%0A++++return+(x+%3C%3C+@truncate(-%25i))+%7C+(x+%3E%3E+@truncate(i))%3B%0A%7D'),l:'5',n:'0',o:'Zig+source+%231',t:'0')),k:51.27826974307506,l:'4',m:100,n:'0',o:'',s:0,t:'0'),(g:!((h:compiler,i:(compiler:ztrunk,filters:(b:'0',binary:'1',binaryObject:'1',commentOnly:'0',debugCalls:'1',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1',verboseDemangling:'0'),flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:2,lang:zig,libs:!(),options:'-O+ReleaseFast+-target+x86_64-linux+-mcpu%3Dznver5+-fomit-frame-pointer',overrides:!(),selection:(endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1),source:1),l:'5',n:'0',o:'+zig+trunk+(Editor+%231)',t:'0')),header:(),k:48.72173025692494,l:'4',m:100.00000000000001,n:'0',o:'',s:0,t:'0')),l:'2',m:100,n:'0',o:'',t:'0')),version:4) [LLVM Godbolt](https://llvm.godbo.lt/#z:OYLghAFBqd5QCxAYwPYBMCmBRdBLAF1QCcAaPECAMzwBtMA7AQwFtMQByARg9KtQYEAysib0QXACx8BBAKoBnTAAUAHpwAMvAFYTStJg1C1aANxakl9ZATwDKjdAGFUtAK4sGexwBk8DTAA5DwAjTGIQAGYANlIAB1QFQjsGF3dPPQSk2wE/AOCWMIiYy0xrHIYhAiZiAjSPLy5S8pSqmoI8oNDwqNiFatr6jKb%2B9s6Cot6ASktUN2Jkdg4sGgCAanQFVAB9WlRRWjWAUkinACY11TW8aMkT7DWAAX5UCBPzy%2Bvb%2B%2BOzgFYNKRjqcLlcbndIg8jv8uFM1nsDts3AxmGx0NsmOh0MRjgB2ABCRw0AEFsIJiABPEBE4lrX5/C4nAAiawIxGRyGBHzB30h9K4rNQXNBX3uNLp0L%2BkWBLIUbhCws%2BNx%2BAC9wqh/MkxHg1WR6WdxfTJDK1mrVARFTyfpLpURLV8IdhDZK/iaFAhDu8ReDrf9AUbnf9oibDOh7T6%2BZKmmsFHEDBaIOC1tFIlNA39cSbaO6cV6lbyoX6gZLoumABwmkjhgv0v7F/64w3ETAWvM8x30ss0o64pndkkrfyYDZbXb7MRc43tn6PEI1N6nKcO30A4uL/MdqNwhFiJEo1iYdGY7F4wkkslsqnpxmRFlsjmTjcrgV295L5WQ9PS5kx%2BWPq18rqGoMFqtA6uE9YMumxo/marbrgBhZSoK/7Lp%2BJISv8ro/u6noIWhSH%2BpKdwYfSwY/qGqERkh0axvGayJrcyapummY/tmCC5vh1H0kRQblpWXFOO%2BNYupBjakc28HCU%2Bkb/F2JI9n2ikDpgqzDpsOw7nhTgVtOfKzkwKoLrpsmEWupn6TR27jrQe6ooeGJYrmBI0helLUqRko3ne7IMJy7x6QR/IoYF%2BZil5/zfrev4KmFiGmuqmq2NquqQQakV/DBMVwVyQUfkhtpCvFBHpthMW4XlZm8ZBJG0mRIYMGGJU8VGQJ0UwCZJimaaZWxMUcUJ%2BWiUWZECT%2BVYtSNfx1vSEn1VJVVWZ23a9jSHAzLQnB/LwXgcFopCoJwPg%2BAAagAsmsACSABKMZzAsw7QpEPCkAQmgbTMADWIBZQAdJIuIAJwaEDuJZYDINg9IW0cJIu0fYdnC8AoICAu9%2B0baQcCwEgaAsHEdAQRQED44T9ARCYNh%2BV9fB0AQ4SoxAISIyE/g1FS3C8GzzCUgA8iE2iYDYnCvfjbCCHzDC0JzB1YCEbjAE4YjZqLvBYCwhjAOImOkPgzY2HgpiYKjuuYKowtuAzavkIIZSI2BITEBzLhYIjbJ4CwNvG8QISJJgTKYJrRhgUYH0zFQBjAAop14JgADufNxIwNv8IIIhiOwUgyIIigqOouu6E0BhhyY5j6HgISo5AMyoHEFSmwAtHz0qNywyBxG4zIqgwPs7YdPvEHgWDVxAMxWMLFQOE1QyNKQvhDhMPRNFkyQCLPmSJGvDDjN0EQjGUk%2BtAMdSuA0egT4bAhtLUu%2BFMvlgnxvIwn3fkywrM8yLBIm3bQjutHQ4PCWgnIID3gYF9OEEBcCEEmmcF6UxeAYy0FMb6IBJB/D%2BgyUGGgzgYOwSDM4sRYbw1IF7LgGhAR7QOoAlGaM3rh1/hwM4/8aHIwYZjVBpAfZJHsJIIAA%3D%3D)

define dso_local &lt;2 x i64&gt; @<!-- -->foo(&lt;2 x i64&gt; %0, &lt;2 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;2 x i64&gt; %1 to &lt;2 x i6&gt;
  %3 = sub &lt;2 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;2 x i6&gt; %3 to &lt;2 x i64&gt;
  %5 = shl &lt;2 x i64&gt; %0, %4
  %6 = and &lt;2 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;2 x i64&gt; %0, %6
  %8 = or &lt;2 x i64&gt; %5, %7
  ret &lt;2 x i64&gt; %8
}

define dso_local &lt;4 x i64&gt; @<!-- -->bar(&lt;4 x i64&gt; %0, &lt;4 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;4 x i64&gt; %1 to &lt;4 x i6&gt;
  %3 = sub &lt;4 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;4 x i6&gt; %3 to &lt;4 x i64&gt;
  %5 = shl &lt;4 x i64&gt; %0, %4
  %6 = and &lt;4 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;4 x i64&gt; %0, %6
  %8 = or &lt;4 x i64&gt; %5, %7
  ret &lt;4 x i64&gt; %8
}

define dso_local &lt;8 x i64&gt; @<!-- -->baz(&lt;8 x i64&gt; %0, &lt;8 x i64&gt; %1) local_unnamed_addr {
Entry:
  %2 = trunc &lt;8 x i64&gt; %1 to &lt;8 x i6&gt;
  %3 = sub &lt;8 x i6&gt; zeroinitializer, %2
  %4 = zext &lt;8 x i6&gt; %3 to &lt;8 x i64&gt;
  %5 = shl &lt;8 x i64&gt; %0, %4
  %6 = and &lt;8 x i64&gt; %1, splat (i64 63)
  %7 = lshr &lt;8 x i64&gt; %0, %6
  %8 = or &lt;8 x i64&gt; %5, %7
  ret &lt;8 x i64&gt; %8
}

Emit:

foo:
        vprorvq xmm0, xmm0, xmm1
        ret

.LCPI1_0:
        .quad   63
bar:
        vpbroadcastq    ymm4, qword ptr [rip + .LCPI1_0]
        vpmovqd xmm2, ymm1
        vpxor   xmm3, xmm3, xmm3
        vpsubd  xmm2, xmm3, xmm2
        vpmovzxdq       ymm2, xmm2
        vpand   ymm2, ymm2, ymm4
        vpand   ymm1, ymm1, ymm4
        vpsllvq ymm2, ymm0, ymm2
        vpsrlvq ymm0, ymm0, ymm1
        vpor    ymm0, ymm2, ymm0
        ret

.LCPI2_0:
        .quad   63
baz:
        vpmovqw xmm2, zmm1
        vpbroadcastq    zmm4, qword ptr [rip + .LCPI2_0]
        vpxor   xmm3, xmm3, xmm3
        vpsubw  xmm2, xmm3, xmm2
        vpmovzxwq       zmm2, xmm2
        vpandq  zmm1, zmm1, zmm4
        vpandq  zmm2, zmm2, zmm4
        vpsllvq zmm2, zmm0, zmm2
        vpsrlvq zmm0, zmm0, zmm1
        vporq   zmm0, zmm2, zmm0
        ret

[Susikrishna]

I would like to work on this issue.