Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 1 | #!/usr/bin/env perl |
| 2 | # |
| 3 | # ==================================================================== |
| 4 | # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL |
| 5 | # project. The module is, however, dual licensed under OpenSSL and |
| 6 | # CRYPTOGAMS licenses depending on where you obtain it. For further |
| 7 | # details see http://www.openssl.org/~appro/cryptogams/. |
| 8 | # ==================================================================== |
| 9 | # |
| 10 | # sha1_block procedure for x86_64. |
| 11 | # |
| 12 | # It was brought to my attention that on EM64T compiler-generated code |
| 13 | # was far behind 32-bit assembler implementation. This is unlike on |
| 14 | # Opteron where compiler-generated code was only 15% behind 32-bit |
| 15 | # assembler, which originally made it hard to motivate the effort. |
| 16 | # There was suggestion to mechanically translate 32-bit code, but I |
| 17 | # dismissed it, reasoning that x86_64 offers enough register bank |
| 18 | # capacity to fully utilize SHA-1 parallelism. Therefore this fresh |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 19 | # implementation:-) However! While 64-bit code does performs better |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 20 | # on Opteron, I failed to beat 32-bit assembler on EM64T core. Well, |
| 21 | # x86_64 does offer larger *addressable* bank, but out-of-order core |
| 22 | # reaches for even more registers through dynamic aliasing, and EM64T |
| 23 | # core must have managed to run-time optimize even 32-bit code just as |
| 24 | # good as 64-bit one. Performance improvement is summarized in the |
| 25 | # following table: |
| 26 | # |
| 27 | # gcc 3.4 32-bit asm cycles/byte |
| 28 | # Opteron +45% +20% 6.8 |
| 29 | # Xeon P4 +65% +0% 9.9 |
| 30 | # Core2 +60% +10% 7.0 |
| 31 | |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 32 | $flavour = shift; |
| 33 | $output = shift; |
| 34 | if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } |
| 35 | |
| 36 | $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| 37 | |
| 38 | $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| 39 | ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| 40 | ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| 41 | die "can't locate x86_64-xlate.pl"; |
| 42 | |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 43 | open STDOUT,"| $^X $xlate $flavour $output"; |
| 44 | |
| 45 | $ctx="%rdi"; # 1st arg |
| 46 | $inp="%rsi"; # 2nd arg |
| 47 | $num="%rdx"; # 3rd arg |
| 48 | |
| 49 | # reassign arguments in order to produce more compact code |
| 50 | $ctx="%r8"; |
| 51 | $inp="%r9"; |
| 52 | $num="%r10"; |
| 53 | |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 54 | $xi="%eax"; |
| 55 | $t0="%ebx"; |
| 56 | $t1="%ecx"; |
| 57 | $A="%edx"; |
| 58 | $B="%esi"; |
| 59 | $C="%edi"; |
| 60 | $D="%ebp"; |
| 61 | $E="%r11d"; |
| 62 | $T="%r12d"; |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 63 | |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 64 | @V=($A,$B,$C,$D,$E,$T); |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 65 | |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 66 | sub PROLOGUE { |
| 67 | my $func=shift; |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 68 | $code.=<<___; |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 69 | .globl $func |
| 70 | .type $func,\@function,3 |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 71 | .align 16 |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 72 | $func: |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 73 | push %rbx |
| 74 | push %rbp |
| 75 | push %r12 |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 76 | mov %rsp,%r11 |
| 77 | mov %rdi,$ctx # reassigned argument |
| 78 | sub \$`8+16*4`,%rsp |
| 79 | mov %rsi,$inp # reassigned argument |
| 80 | and \$-64,%rsp |
| 81 | mov %rdx,$num # reassigned argument |
| 82 | mov %r11,`16*4`(%rsp) |
| 83 | .Lprologue: |
| 84 | |
| 85 | mov 0($ctx),$A |
| 86 | mov 4($ctx),$B |
| 87 | mov 8($ctx),$C |
| 88 | mov 12($ctx),$D |
| 89 | mov 16($ctx),$E |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 90 | ___ |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 91 | } |
| 92 | |
| 93 | sub EPILOGUE { |
| 94 | my $func=shift; |
| 95 | $code.=<<___; |
| 96 | mov `16*4`(%rsp),%rsi |
| 97 | mov (%rsi),%r12 |
| 98 | mov 8(%rsi),%rbp |
| 99 | mov 16(%rsi),%rbx |
| 100 | lea 24(%rsi),%rsp |
| 101 | .Lepilogue: |
| 102 | ret |
| 103 | .size $func,.-$func |
| 104 | ___ |
| 105 | } |
| 106 | |
| 107 | sub BODY_00_19 { |
| 108 | my ($i,$a,$b,$c,$d,$e,$f,$host)=@_; |
| 109 | my $j=$i+1; |
| 110 | $code.=<<___ if ($i==0); |
| 111 | mov `4*$i`($inp),$xi |
| 112 | `"bswap $xi" if(!defined($host))` |
| 113 | mov $xi,`4*$i`(%rsp) |
| 114 | ___ |
| 115 | $code.=<<___ if ($i<15); |
| 116 | lea 0x5a827999($xi,$e),$f |
| 117 | mov $c,$t0 |
| 118 | mov `4*$j`($inp),$xi |
| 119 | mov $a,$e |
| 120 | xor $d,$t0 |
| 121 | `"bswap $xi" if(!defined($host))` |
| 122 | rol \$5,$e |
| 123 | and $b,$t0 |
| 124 | mov $xi,`4*$j`(%rsp) |
| 125 | add $e,$f |
| 126 | xor $d,$t0 |
| 127 | rol \$30,$b |
| 128 | add $t0,$f |
| 129 | ___ |
| 130 | $code.=<<___ if ($i>=15); |
| 131 | lea 0x5a827999($xi,$e),$f |
| 132 | mov `4*($j%16)`(%rsp),$xi |
| 133 | mov $c,$t0 |
| 134 | mov $a,$e |
| 135 | xor `4*(($j+2)%16)`(%rsp),$xi |
| 136 | xor $d,$t0 |
| 137 | rol \$5,$e |
| 138 | xor `4*(($j+8)%16)`(%rsp),$xi |
| 139 | and $b,$t0 |
| 140 | add $e,$f |
| 141 | xor `4*(($j+13)%16)`(%rsp),$xi |
| 142 | xor $d,$t0 |
| 143 | rol \$30,$b |
| 144 | add $t0,$f |
| 145 | rol \$1,$xi |
| 146 | mov $xi,`4*($j%16)`(%rsp) |
| 147 | ___ |
| 148 | } |
| 149 | |
| 150 | sub BODY_20_39 { |
| 151 | my ($i,$a,$b,$c,$d,$e,$f)=@_; |
| 152 | my $j=$i+1; |
| 153 | my $K=($i<40)?0x6ed9eba1:0xca62c1d6; |
| 154 | $code.=<<___ if ($i<79); |
| 155 | lea $K($xi,$e),$f |
| 156 | mov `4*($j%16)`(%rsp),$xi |
| 157 | mov $c,$t0 |
| 158 | mov $a,$e |
| 159 | xor `4*(($j+2)%16)`(%rsp),$xi |
| 160 | xor $b,$t0 |
| 161 | rol \$5,$e |
| 162 | xor `4*(($j+8)%16)`(%rsp),$xi |
| 163 | xor $d,$t0 |
| 164 | add $e,$f |
| 165 | xor `4*(($j+13)%16)`(%rsp),$xi |
| 166 | rol \$30,$b |
| 167 | add $t0,$f |
| 168 | rol \$1,$xi |
| 169 | ___ |
| 170 | $code.=<<___ if ($i<76); |
| 171 | mov $xi,`4*($j%16)`(%rsp) |
| 172 | ___ |
| 173 | $code.=<<___ if ($i==79); |
| 174 | lea $K($xi,$e),$f |
| 175 | mov $c,$t0 |
| 176 | mov $a,$e |
| 177 | xor $b,$t0 |
| 178 | rol \$5,$e |
| 179 | xor $d,$t0 |
| 180 | add $e,$f |
| 181 | rol \$30,$b |
| 182 | add $t0,$f |
| 183 | ___ |
| 184 | } |
| 185 | |
| 186 | sub BODY_40_59 { |
| 187 | my ($i,$a,$b,$c,$d,$e,$f)=@_; |
| 188 | my $j=$i+1; |
| 189 | $code.=<<___; |
| 190 | lea 0x8f1bbcdc($xi,$e),$f |
| 191 | mov `4*($j%16)`(%rsp),$xi |
| 192 | mov $b,$t0 |
| 193 | mov $b,$t1 |
| 194 | xor `4*(($j+2)%16)`(%rsp),$xi |
| 195 | mov $a,$e |
| 196 | and $c,$t0 |
| 197 | xor `4*(($j+8)%16)`(%rsp),$xi |
| 198 | or $c,$t1 |
| 199 | rol \$5,$e |
| 200 | xor `4*(($j+13)%16)`(%rsp),$xi |
| 201 | and $d,$t1 |
| 202 | add $e,$f |
| 203 | rol \$1,$xi |
| 204 | or $t1,$t0 |
| 205 | rol \$30,$b |
| 206 | mov $xi,`4*($j%16)`(%rsp) |
| 207 | add $t0,$f |
| 208 | ___ |
| 209 | } |
| 210 | |
| 211 | $code=".text\n"; |
| 212 | |
| 213 | &PROLOGUE("sha1_block_data_order"); |
| 214 | $code.=".align 4\n.Lloop:\n"; |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 215 | for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); } |
| 216 | for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } |
| 217 | for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } |
| 218 | for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } |
| 219 | $code.=<<___; |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 220 | add 0($ctx),$E |
| 221 | add 4($ctx),$T |
| 222 | add 8($ctx),$A |
| 223 | add 12($ctx),$B |
| 224 | add 16($ctx),$C |
| 225 | mov $E,0($ctx) |
| 226 | mov $T,4($ctx) |
| 227 | mov $A,8($ctx) |
| 228 | mov $B,12($ctx) |
| 229 | mov $C,16($ctx) |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 230 | |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 231 | xchg $E,$A # mov $E,$A |
| 232 | xchg $T,$B # mov $T,$B |
| 233 | xchg $E,$C # mov $A,$C |
| 234 | xchg $T,$D # mov $B,$D |
| 235 | # mov $C,$E |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 236 | lea `16*4`($inp),$inp |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 237 | sub \$1,$num |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 238 | jnz .Lloop |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 239 | ___ |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 240 | &EPILOGUE("sha1_block_data_order"); |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 241 | $code.=<<___; |
| 242 | .asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>" |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 243 | .align 16 |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 244 | ___ |
| 245 | |
| 246 | # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, |
| 247 | # CONTEXT *context,DISPATCHER_CONTEXT *disp) |
| 248 | if ($win64) { |
| 249 | $rec="%rcx"; |
| 250 | $frame="%rdx"; |
| 251 | $context="%r8"; |
| 252 | $disp="%r9"; |
| 253 | |
| 254 | $code.=<<___; |
| 255 | .extern __imp_RtlVirtualUnwind |
| 256 | .type se_handler,\@abi-omnipotent |
| 257 | .align 16 |
| 258 | se_handler: |
| 259 | push %rsi |
| 260 | push %rdi |
| 261 | push %rbx |
| 262 | push %rbp |
| 263 | push %r12 |
| 264 | push %r13 |
| 265 | push %r14 |
| 266 | push %r15 |
| 267 | pushfq |
| 268 | sub \$64,%rsp |
| 269 | |
| 270 | mov 120($context),%rax # pull context->Rax |
| 271 | mov 248($context),%rbx # pull context->Rip |
| 272 | |
| 273 | lea .Lprologue(%rip),%r10 |
| 274 | cmp %r10,%rbx # context->Rip<.Lprologue |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 275 | jb .Lin_prologue |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 276 | |
| 277 | mov 152($context),%rax # pull context->Rsp |
| 278 | |
| 279 | lea .Lepilogue(%rip),%r10 |
| 280 | cmp %r10,%rbx # context->Rip>=.Lepilogue |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 281 | jae .Lin_prologue |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 282 | |
| 283 | mov `16*4`(%rax),%rax # pull saved stack pointer |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 284 | lea 24(%rax),%rax |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 285 | |
| 286 | mov -8(%rax),%rbx |
| 287 | mov -16(%rax),%rbp |
| 288 | mov -24(%rax),%r12 |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 289 | mov %rbx,144($context) # restore context->Rbx |
| 290 | mov %rbp,160($context) # restore context->Rbp |
| 291 | mov %r12,216($context) # restore context->R12 |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 292 | |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 293 | .Lin_prologue: |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 294 | mov 8(%rax),%rdi |
| 295 | mov 16(%rax),%rsi |
| 296 | mov %rax,152($context) # restore context->Rsp |
| 297 | mov %rsi,168($context) # restore context->Rsi |
| 298 | mov %rdi,176($context) # restore context->Rdi |
| 299 | |
| 300 | mov 40($disp),%rdi # disp->ContextRecord |
| 301 | mov $context,%rsi # context |
| 302 | mov \$154,%ecx # sizeof(CONTEXT) |
| 303 | .long 0xa548f3fc # cld; rep movsq |
| 304 | |
| 305 | mov $disp,%rsi |
| 306 | xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| 307 | mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| 308 | mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| 309 | mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| 310 | mov 40(%rsi),%r10 # disp->ContextRecord |
| 311 | lea 56(%rsi),%r11 # &disp->HandlerData |
| 312 | lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| 313 | mov %r10,32(%rsp) # arg5 |
| 314 | mov %r11,40(%rsp) # arg6 |
| 315 | mov %r12,48(%rsp) # arg7 |
| 316 | mov %rcx,56(%rsp) # arg8, (NULL) |
| 317 | call *__imp_RtlVirtualUnwind(%rip) |
| 318 | |
| 319 | mov \$1,%eax # ExceptionContinueSearch |
| 320 | add \$64,%rsp |
| 321 | popfq |
| 322 | pop %r15 |
| 323 | pop %r14 |
| 324 | pop %r13 |
| 325 | pop %r12 |
| 326 | pop %rbp |
| 327 | pop %rbx |
| 328 | pop %rdi |
| 329 | pop %rsi |
| 330 | ret |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 331 | .size se_handler,.-se_handler |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 332 | |
| 333 | .section .pdata |
| 334 | .align 4 |
| 335 | .rva .LSEH_begin_sha1_block_data_order |
| 336 | .rva .LSEH_end_sha1_block_data_order |
| 337 | .rva .LSEH_info_sha1_block_data_order |
Alexandre Savard | 7541067 | 2012-08-08 09:50:01 -0400 | [diff] [blame^] | 338 | |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 339 | .section .xdata |
| 340 | .align 8 |
| 341 | .LSEH_info_sha1_block_data_order: |
| 342 | .byte 9,0,0,0 |
| 343 | .rva se_handler |
Alexandre Savard | 1b09e31 | 2012-08-07 20:33:29 -0400 | [diff] [blame] | 344 | ___ |
| 345 | } |
| 346 | |
| 347 | #################################################################### |
| 348 | |
| 349 | $code =~ s/\`([^\`]*)\`/eval $1/gem; |
| 350 | print $code; |
| 351 | close STDOUT; |