diff --git a/makefile b/makefile index 55e1abd..d0a969c 100644 --- a/makefile +++ b/makefile @@ -11,7 +11,7 @@ SRCDIR=src OBJDIR=obj LDFLAGS=-lpthread TOBJS=$(addprefix $(OBJDIR)/,instructionsPortable.o TestAluFpu.o) -ROBJS=$(addprefix $(OBJDIR)/,argon2_core.o argon2_ref.o AssemblyGeneratorX86.o blake2b.o CompiledVirtualMachine.o dataset.o JitCompilerX86.o instructionsPortable.o Instruction.o InterpretedVirtualMachine.o main.o Program.o softAes.o VirtualMachine.o t1ha2.o Cache.o virtualMemory.o) +ROBJS=$(addprefix $(OBJDIR)/,argon2_core.o argon2_ref.o AssemblyGeneratorX86.o blake2b.o CompiledVirtualMachine.o dataset.o JitCompilerX86.o instructionsPortable.o Instruction.o InterpretedVirtualMachine.o main.o Program.o softAes.o VirtualMachine.o t1ha2.o Cache.o virtualMemory.o divideByConstantCodegen.o) ifeq ($(PLATFORM),x86_64) ROBJS += $(OBJDIR)/JitCompilerX86-static.o endif @@ -57,6 +57,9 @@ $(OBJDIR)/CompiledVirtualMachine.o: $(addprefix $(SRCDIR)/,CompiledVirtualMachin $(OBJDIR)/dataset.o: $(addprefix $(SRCDIR)/,dataset.cpp common.hpp Pcg32.hpp) | $(OBJDIR) $(CXX) $(CXXFLAGS) -c $(SRCDIR)/dataset.cpp -o $@ +$(OBJDIR)/divideByConstantCodegen.o: $(addprefix $(SRCDIR)/,divideByConstantCodegen.c divideByConstantCodegen.h) | $(OBJDIR) + $(CC) $(CCFLAGS) -c $(SRCDIR)/divideByConstantCodegen.c -o $@ + $(OBJDIR)/JitCompilerX86.o: $(addprefix $(SRCDIR)/,JitCompilerX86.cpp JitCompilerX86.hpp Instruction.hpp instructionWeights.hpp) | $(OBJDIR) $(CXX) $(CXXFLAGS) -c $(SRCDIR)/JitCompilerX86.cpp -o $@ diff --git a/src/AssemblyGeneratorX86.cpp b/src/AssemblyGeneratorX86.cpp index 1fbf2f2..9389634 100644 --- a/src/AssemblyGeneratorX86.cpp +++ b/src/AssemblyGeneratorX86.cpp @@ -17,10 +17,14 @@ You should have received a copy of the GNU General Public License along with RandomX. If not, see. */ //#define TRACE +//#define MAGIC_DIVISION #include "AssemblyGeneratorX86.hpp" #include "Pcg32.hpp" #include "common.hpp" #include "instructions.hpp" +#ifdef MAGIC_DIVISION +#include "divideByConstantCodegen.h" +#endif namespace RandomX { @@ -315,34 +319,118 @@ namespace RandomX { void AssemblyGeneratorX86::h_DIV_64(Instruction& instr, int i) { genar(instr, i); if ((instr.locb & 7) >= 6) { +#ifdef MAGIC_DIVISION + if (instr.imm32 != 0) { + uint32_t divisor = instr.imm32; + asmCode << "\t; magic divide by " << divisor << std::endl; + if (divisor & (divisor - 1)) { + magicu_info mi = compute_unsigned_magic_info(divisor, sizeof(uint64_t) * 8); + if (mi.pre_shift > 0) + asmCode << "\tshr rax, " << mi.pre_shift << std::endl; + if (mi.increment) { + asmCode << "\tadd rax, 1" << std::endl; + asmCode << "\tsbb rax, 0" << std::endl; + } + asmCode << "\tmov rcx, " << mi.multiplier << std::endl; + asmCode << "\tmul rcx" << std::endl; + asmCode << "\tmov rax, rdx" << std::endl; + if (mi.post_shift > 0) + asmCode << "\tshr rax, " << mi.post_shift << std::endl; + } + else { //divisor is a power of two + int shift = 0; + while (divisor >>= 1) + ++shift; + if(shift > 0) + asmCode << "\tshr rax, " << shift << std::endl; + } + } +#else if (instr.imm32 == 0) { asmCode << "\tmov ecx, 1" << std::endl; } else { asmCode << "\tmov ecx, " << instr.imm32 << std::endl; } +#endif } else { asmCode << "\tmov ecx, 1" << std::endl; asmCode << "\tmov edx, " << regR32[instr.regb % RegistersCount] << std::endl; asmCode << "\ttest edx, edx" << std::endl; asmCode << "\tcmovne ecx, edx" << std::endl; +#ifdef MAGIC_DIVISION + asmCode << "\txor edx, edx" << std::endl; + asmCode << "\tdiv rcx" << std::endl; +#endif } +#ifndef MAGIC_DIVISION asmCode << "\txor edx, edx" << std::endl; asmCode << "\tdiv rcx" << std::endl; +#endif gencr(instr); } void AssemblyGeneratorX86::h_IDIV_64(Instruction& instr, int i) { genar(instr, i); +#ifdef MAGIC_DIVISION + if ((instr.locb & 7) >= 6) { + int64_t divisor = instr.imm32; + asmCode << "\t; magic divide by " << divisor << std::endl; + if ((divisor & -divisor) == divisor || (divisor & -divisor) == -divisor) { + // +/- power of two + bool negative = divisor < 0; + if (negative) + divisor = -divisor; + int shift = 0; + uint64_t unsignedDivisor = divisor; + while (unsignedDivisor >>= 1) + ++shift; + if (shift > 0) { + asmCode << "\tmov rcx, rax" << std::endl; + asmCode << "\tsar rcx, 63" << std::endl; + uint32_t mask = (1ULL << shift) + 0xFFFFFFFF; + asmCode << "\tand ecx, 0" << std::hex << mask << std::dec << "h" << std::endl; + asmCode << "\tadd rax, rcx" << std::endl; + asmCode << "\tsar rax, " << shift << std::endl; + } + if(negative) + asmCode << "\tneg rax" << std::endl; + } else if(divisor != 0) { + magics_info mi = compute_signed_magic_info(divisor); + if ((divisor >= 0) != (mi.multiplier >= 0)) + asmCode << "\tmov rcx, rax" << std::endl; + asmCode << "\tmov rdx, " << mi.multiplier << std::endl; + asmCode << "\timul rdx" << std::endl; + asmCode << "\tmov rax, rdx" << std::endl; + asmCode << "\txor edx, edx" << std::endl; + bool haveSF = false; + if (divisor > 0 && mi.multiplier < 0) { + asmCode << "\tadd rax, rcx" << std::endl; + haveSF = true; + } + if (divisor < 0 && mi.multiplier > 0) { + asmCode << "\tsub rax, rcx" << std::endl; + haveSF = true; + } + if (mi.shift > 0) { + asmCode << "\tsar rax, " << mi.shift << std::endl; + haveSF = true; + } + if (!haveSF) + asmCode << "\ttest rax, rax" << std::endl; + asmCode << "\tsets dl" << std::endl; + asmCode << "\tadd rax, rdx" << std::endl; + } + } + else { +#endif asmCode << "\tmov edx, "; genbr132(instr); asmCode << "\tcmp edx, -1" << std::endl; asmCode << "\tjne short safe_idiv_" << i << std::endl; - asmCode << "\tmov rcx, rax" << std::endl; - asmCode << "\trol rcx, 1" << std::endl; - asmCode << "\tdec rcx" << std::endl; - asmCode << "\tjz short result_idiv_" << i << std::endl; + asmCode << "\tneg rax" << std::endl; + asmCode << "\tjmp short result_idiv_" << i << std::endl; asmCode << "safe_idiv_" << i << ":" << std::endl; asmCode << "\tmov ecx, 1" << std::endl; asmCode << "\ttest edx, edx" << std::endl; @@ -351,6 +439,9 @@ namespace RandomX { asmCode << "\tcqo" << std::endl; asmCode << "\tidiv rcx" << std::endl; asmCode << "result_idiv_" << i << ":" << std::endl; +#ifdef MAGIC_DIVISION + } +#endif gencr(instr); } diff --git a/src/divideByConstantCodegen.c b/src/divideByConstantCodegen.c new file mode 100644 index 0000000..4b06712 --- /dev/null +++ b/src/divideByConstantCodegen.c @@ -0,0 +1,169 @@ +/* + Reference implementations of computing and using the "magic number" approach to dividing + by constants, including codegen instructions. The unsigned division incorporates the + "round down" optimization per ridiculous_fish. + + This is free and unencumbered software. Any copyright is dedicated to the Public Domain. +*/ + +#include //for CHAR_BIT +#include + +#include "divideByConstantCodegen.h" + +struct magicu_info compute_unsigned_magic_info(uint D, unsigned num_bits) { + + //The numerator must fit in a uint + assert(num_bits > 0 && num_bits <= sizeof(uint) * CHAR_BIT); + + // D must be larger than zero and not a power of 2 + assert(D & (D - 1)); + + // The eventual result + struct magicu_info result; + + // Bits in a uint + const unsigned UINT_BITS = sizeof(uint) * CHAR_BIT; + + // The extra shift implicit in the difference between UINT_BITS and num_bits + const unsigned extra_shift = UINT_BITS - num_bits; + + // The initial power of 2 is one less than the first one that can possibly work + const uint initial_power_of_2 = (uint)1 << (UINT_BITS - 1); + + // The remainder and quotient of our power of 2 divided by d + uint quotient = initial_power_of_2 / D, remainder = initial_power_of_2 % D; + + // ceil(log_2 D) + unsigned ceil_log_2_D; + + // The magic info for the variant "round down" algorithm + uint down_multiplier = 0; + unsigned down_exponent = 0; + int has_magic_down = 0; + + // Compute ceil(log_2 D) + ceil_log_2_D = 0; + uint tmp; + for (tmp = D; tmp > 0; tmp >>= 1) + ceil_log_2_D += 1; + + + // Begin a loop that increments the exponent, until we find a power of 2 that works. + unsigned exponent; + for (exponent = 0; ; exponent++) { + // Quotient and remainder is from previous exponent; compute it for this exponent. + if (remainder >= D - remainder) { + // Doubling remainder will wrap around D + quotient = quotient * 2 + 1; + remainder = remainder * 2 - D; + } + else { + // Remainder will not wrap + quotient = quotient * 2; + remainder = remainder * 2; + } + + // We're done if this exponent works for the round_up algorithm. + // Note that exponent may be larger than the maximum shift supported, + // so the check for >= ceil_log_2_D is critical. + if ((exponent + extra_shift >= ceil_log_2_D) || (D - remainder) <= ((uint)1 << (exponent + extra_shift))) + break; + + // Set magic_down if we have not set it yet and this exponent works for the round_down algorithm + if (!has_magic_down && remainder <= ((uint)1 << (exponent + extra_shift))) { + has_magic_down = 1; + down_multiplier = quotient; + down_exponent = exponent; + } + } + + if (exponent < ceil_log_2_D) { + // magic_up is efficient + result.multiplier = quotient + 1; + result.pre_shift = 0; + result.post_shift = exponent; + result.increment = 0; + } + else if (D & 1) { + // Odd divisor, so use magic_down, which must have been set + assert(has_magic_down); + result.multiplier = down_multiplier; + result.pre_shift = 0; + result.post_shift = down_exponent; + result.increment = 1; + } + else { + // Even divisor, so use a prefix-shifted dividend + unsigned pre_shift = 0; + uint shifted_D = D; + while ((shifted_D & 1) == 0) { + shifted_D >>= 1; + pre_shift += 1; + } + result = compute_unsigned_magic_info(shifted_D, num_bits - pre_shift); + assert(result.increment == 0 && result.pre_shift == 0); //expect no increment or pre_shift in this path + result.pre_shift = pre_shift; + } + return result; +} + +struct magics_info compute_signed_magic_info(sint D) { + // D must not be zero and must not be a power of 2 (or its negative) + assert(D != 0 && (D & -D) != D && (D & -D) != -D); + + // Our result + struct magics_info result; + + // Bits in an sint + const unsigned SINT_BITS = sizeof(sint) * CHAR_BIT; + + // Absolute value of D (we know D is not the most negative value since that's a power of 2) + const uint abs_d = (D < 0 ? -D : D); + + // The initial power of 2 is one less than the first one that can possibly work + // "two31" in Warren + unsigned exponent = SINT_BITS - 1; + const uint initial_power_of_2 = (uint)1 << exponent; + + // Compute the absolute value of our "test numerator," + // which is the largest dividend whose remainder with d is d-1. + // This is called anc in Warren. + const uint tmp = initial_power_of_2 + (D < 0); + const uint abs_test_numer = tmp - 1 - tmp % abs_d; + + // Initialize our quotients and remainders (q1, r1, q2, r2 in Warren) + uint quotient1 = initial_power_of_2 / abs_test_numer, remainder1 = initial_power_of_2 % abs_test_numer; + uint quotient2 = initial_power_of_2 / abs_d, remainder2 = initial_power_of_2 % abs_d; + uint delta; + + // Begin our loop + do { + // Update the exponent + exponent++; + + // Update quotient1 and remainder1 + quotient1 *= 2; + remainder1 *= 2; + if (remainder1 >= abs_test_numer) { + quotient1 += 1; + remainder1 -= abs_test_numer; + } + + // Update quotient2 and remainder2 + quotient2 *= 2; + remainder2 *= 2; + if (remainder2 >= abs_d) { + quotient2 += 1; + remainder2 -= abs_d; + } + + // Keep going as long as (2**exponent) / abs_d <= delta + delta = abs_d - remainder2; + } while (quotient1 < delta || (quotient1 == delta && remainder1 == 0)); + + result.multiplier = quotient2 + 1; + if (D < 0) result.multiplier = -result.multiplier; + result.shift = exponent - SINT_BITS; + return result; +} diff --git a/src/divideByConstantCodegen.h b/src/divideByConstantCodegen.h new file mode 100644 index 0000000..1ac55e8 --- /dev/null +++ b/src/divideByConstantCodegen.h @@ -0,0 +1,117 @@ +/* +Copyright (c) 2018 tevador + +This file is part of RandomX. + +RandomX is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX. If not, see. +*/ + +#pragma once +#include + +#if defined(__cplusplus) +extern "C" { +#endif + + typedef uint64_t uint; + typedef int64_t sint; + + /* Computes "magic info" for performing signed division by a fixed integer D. + The type 'sint' is assumed to be defined as a signed integer type large enough + to hold both the dividend and the divisor. + Here >> is arithmetic (signed) shift, and >>> is logical shift. + + To emit code for n/d, rounding towards zero, use the following sequence: + + m = compute_signed_magic_info(D) + emit("result = (m.multiplier * n) >> SINT_BITS"); + if d > 0 and m.multiplier < 0: emit("result += n") + if d < 0 and m.multiplier > 0: emit("result -= n") + if m.post_shift > 0: emit("result >>= m.shift") + emit("result += (result < 0)") + + The shifts by SINT_BITS may be "free" if the high half of the full multiply + is put in a separate register. + + The final add can of course be implemented via the sign bit, e.g. + result += (result >>> (SINT_BITS - 1)) + or + result -= (result >> (SINT_BITS - 1)) + + This code is heavily indebted to Hacker's Delight by Henry Warren. + See http://www.hackersdelight.org/HDcode/magic.c.txt + Used with permission from http://www.hackersdelight.org/permissions.htm + */ + + struct magics_info { + sint multiplier; // the "magic number" multiplier + unsigned shift; // shift for the dividend after multiplying + }; + struct magics_info compute_signed_magic_info(sint D); + + + /* Computes "magic info" for performing unsigned division by a fixed positive integer D. + The type 'uint' is assumed to be defined as an unsigned integer type large enough + to hold both the dividend and the divisor. num_bits can be set appropriately if n is + known to be smaller than the largest uint; if this is not known then pass + (sizeof(uint) * CHAR_BIT) for num_bits. + + Assume we have a hardware register of width UINT_BITS, a known constant D which is + not zero and not a power of 2, and a variable n of width num_bits (which may be + up to UINT_BITS). To emit code for n/d, use one of the two following sequences + (here >>> refers to a logical bitshift): + + m = compute_unsigned_magic_info(D, num_bits) + if m.pre_shift > 0: emit("n >>>= m.pre_shift") + if m.increment: emit("n = saturated_increment(n)") + emit("result = (m.multiplier * n) >>> UINT_BITS") + if m.post_shift > 0: emit("result >>>= m.post_shift") + + or + + m = compute_unsigned_magic_info(D, num_bits) + if m.pre_shift > 0: emit("n >>>= m.pre_shift") + emit("result = m.multiplier * n") + if m.increment: emit("result = result + m.multiplier") + emit("result >>>= UINT_BITS") + if m.post_shift > 0: emit("result >>>= m.post_shift") + + The shifts by UINT_BITS may be "free" if the high half of the full multiply + is put in a separate register. + + saturated_increment(n) means "increment n unless it would wrap to 0," i.e. + if n == (1 << UINT_BITS)-1: result = n + else: result = n+1 + A common way to implement this is with the carry bit. For example, on x86: + add 1 + sbb 0 + + Some invariants: + 1: At least one of pre_shift and increment is zero + 2: multiplier is never zero + + This code incorporates the "round down" optimization per ridiculous_fish. + */ + + struct magicu_info { + uint multiplier; // the "magic number" multiplier + unsigned pre_shift; // shift for the dividend before multiplying + unsigned post_shift; //shift for the dividend after multiplying + int increment; // 0 or 1; if set then increment the numerator, using one of the two strategies + }; + struct magicu_info compute_unsigned_magic_info(uint D, unsigned num_bits); + +#if defined(__cplusplus) +} +#endif \ No newline at end of file