More than 64 part4

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */

src/coreclr/jit/compiler.hpp

@@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 // The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
  assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
@@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask)
  return regNum;
 }
 
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+// mask - the register mask
+//
+// Return Value:
+// The number of the first register contained in the mask.
+//
+
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+ assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+ /* Convert the mask to a register number */
+
+ regNumber regNum = (regNumber)BitScanForward(mask);
+
+ return regNum;
+}
+
 //------------------------------------------------------------------------------
 // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
 // register number and also toggle the bit in the `mask`.
@@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)genFirstRegNumFromMask(mask);
 
- mask ^= genRegMask(regNum);
+ mask ^= regNum;
 
  return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+// register number and also toggle the bit in the `mask`.
 // Arguments:
 // mask - the register mask
 //
 // Return Value:
-// The number of the first register contained in the mask.
+// The number of the first register contained in the mask and updates the `mask` to toggle
+// the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
- assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+ assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
  /* Convert the mask to a register number */
 
- regNumber regNum = (regNumber)BitScanForward(mask);
+ regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+ mask ^= genRegMask(regNum);
 
  return regNum;
 }

src/coreclr/jit/lsra.cpp

@@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition)
 {
  RefPosition* nextKill = killRefPosition->nextRefPosition;
 
-regMaskTP killedRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet killedRegs = killRefPosition->registerAssignment;
  while (killedRegs != RBM_NONE)
  {
  regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs);
@@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition)
 {
  // For each physical register that can hold a GC type,
  // if it is occupied by an interval of a GC type, spill that interval.
-regMaskTP candidateRegs = killRefPosition->registerAssignment;
+SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment;
  INDEBUG(bool killedRegs = false);
  while (candidateRegs != RBM_NONE)
  {
@@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar
  {
  // If we're rotating the register locations at block boundaries, try to use
  // the next higher register number of the appropriate register type.
- regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs;
- regNumber firstReg = REG_NA;
- regNumber newReg = REG_NA;
+ SingleTypeRegSet candidateRegs =
+ allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType);
+ regNumber firstReg = REG_NA;
+ regNumber newReg = REG_NA;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);
@@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation()
  // However, we will assert that, at resolution time, no registers contain GC refs.
  {
  DBEXEC(VERBOSE, printf(" "));
-regMaskTP candidateRegs = currentRefPosition.registerAssignment;
+SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment;
  while (candidateRegs != RBM_NONE)
  {
  regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs);

src/coreclr/jit/lsrabuild.cpp

@@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals()
  availableRegCount = REG_INT_COUNT;
  }
 
- if (availableRegCount < (sizeof(regMaskTP) * 8))
+ static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall));
+
+ if (availableRegCount < (sizeof(regMaskSmall) * 8))
+ {
+ // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount
+ actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1);
+ }
+ else if (availableRegCount < (sizeof(regMaskTP) * 8))
  {
- // Mask out the bits that are between 64 ~ availableRegCount
- actualRegistersMask = (1ULL << availableRegCount) - 1;
+ // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount
+ // Subtract one extra for stack.
+ unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1;
+ actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1);
  }
  else
  {
- actualRegistersMask = ~RBM_NONE;
+ actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE);
  }
 
 #ifdef DEBUG

src/coreclr/jit/regset.h

@@ -119,7 +119,7 @@ class RegSet
  bool rsRegsModified(regMaskTP mask) const
  {
  assert(rsModifiedRegsMaskInitialized);
- return (rsModifiedRegsMask & mask) != 0;
+ return (rsModifiedRegsMask & mask).IsNonEmpty();
  }
 
  void verifyRegUsed(regNumber reg);

src/coreclr/jit/target.h

@@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall;
 #define REG_MASK_ALL_FMT "%016llX"
 #endif
 
-typedef regMaskSmall SingleTypeRegSet;
+typedef regMaskSmall SingleTypeRegSet;
+inline SingleTypeRegSet genRegMask(regNumber reg);
+inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
 
 struct regMaskTP
 {
 private:
  regMaskSmall low;
+ regMaskSmall high;
+
 public:
- constexpr regMaskTP(regMaskSmall regMask)
+
+ regMaskTP(regMaskSmall lowMask, regMaskSmall highMask)
+ : low(lowMask)
+ , high(highMask)
+ {
+ }
+
+ regMaskTP(regMaskSmall regMask)
  : low(regMask)
  {
  }
@@ -278,12 +289,12 @@ struct regMaskTP
  return low;
  }
 
- bool IsEmpty()
+ bool IsEmpty() const
  {
  return low == RBM_NONE;
  }
 
- bool IsNonEmpty()
+ bool IsNonEmpty() const
  {
  return !IsEmpty();
  }
@@ -296,50 +307,52 @@ struct regMaskTP
  void RemoveRegNumFromMask(regNumber reg);
 
  bool IsRegNumInMask(regNumber reg);
+
+ void operator|=(const regMaskTP& second)
+ {
+ low |= second.getLow();
+ }
+
+ void operator^=(const regMaskTP& second)
+ {
+ low ^= second.getLow();
+ }
+
+ void operator^=(const regNumber reg)
+ {
+ low ^= genRegMask(reg);
+ }
+
+ void operator&=(const regMaskTP& second)
+ {
+ low &= second.getLow();
+ }
 };
 
-static regMaskTP operator^(regMaskTP first, regMaskTP second)
+static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() ^ second.getLow());
  return result;
 }
 
-static regMaskTP operator&(regMaskTP first, regMaskTP second)
+static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() & second.getLow());
  return result;
 }
 
-static regMaskTP operator|(regMaskTP first, regMaskTP second)
+static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second)
 {
  regMaskTP result(first.getLow() | second.getLow());
  return result;
 }
 
-static regMaskTP& operator|=(regMaskTP& first, regMaskTP second)
-{
- first = first | second;
- return first;
-}
-
-static regMaskTP& operator^=(regMaskTP& first, regMaskTP second)
-{
- first = first ^ second;
- return first;
-}
-
-static regMaskTP& operator&=(regMaskTP& first, regMaskTP second)
-{
- first = first & second;
- return first;
-}
-
-static bool operator==(regMaskTP first, regMaskTP second)
+static bool operator==(const regMaskTP& first, const regMaskTP& second)
 {
  return (first.getLow() == second.getLow());
 }
 
-static bool operator!=(regMaskTP first, regMaskTP second)
+static bool operator!=(const regMaskTP& first, const regMaskTP& second)
 {
  return !(first == second);
 }
@@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b)
 }
 #endif
 
-static regMaskTP operator~(regMaskTP first)
+static regMaskTP operator~(const regMaskTP& first)
 {
  regMaskTP result(~first.getLow());
  return result;
 }
 
-static uint32_t PopCount(regMaskTP value)
+static uint32_t PopCount(const regMaskTP& value)
 {
  return BitOperations::PopCount(value.getLow());
 }
 
-static uint32_t BitScanForward(regMaskTP mask)
+static uint32_t BitScanForward(const regMaskTP& mask)
 {
  return BitOperations::BitScanForward(mask.getLow());
 }
@@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg)
 }
 #endif
 
-inline SingleTypeRegSet genRegMask(regNumber reg);
-inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE));
-
 /*****************************************************************************
  * Return true if the register number is valid
  */