Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/Dialect/CheddarGPU/IR/CheddarGPUAttributes.td
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ def CheddarGPU_FormAttr : EnumAttr<CheddarGPU_Dialect, CheddarGPU_FormEnum, "for

def CheddarGPU_INTTEpilogueAttr : EnumAttr<CheddarGPU_Dialect, CheddarGPU_INTTEpilogueEnum, "intt_epilogue">;

// === Hoist Phase Kind Enum Attribute ===

def CheddarGPU_HoistPhaseKindAttr : EnumAttr<CheddarGPU_Dialect, CheddarGPU_HoistPhaseKindEnum, "hoist_phase_kind">;

// === Params Attribute ===

def CheddarGPU_ParamsAttr : CheddarGPU_Attr<"Params", "params"> {
Expand Down
54 changes: 54 additions & 0 deletions lib/Dialect/CheddarGPU/IR/CheddarGPUConstraints.td
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ def CheddarGPU_PolyOrLimbBuffer : AnyTypeOf<
def CheddarGPU_BConvTable : CheddarGPU_StaticMemRefOfRank<[I32, I64], 2,
"statically shaped 2D memref<D x S x iW> basis-conversion table (W = 32 or 64)">;

// 1D statically shaped byte slab of i8 bytes.
def CheddarGPU_ByteSlab : CheddarGPU_StaticMemRefOfRank<[AnyI8], 1,
"statically shaped 1D memref of i8 bytes">;

// 2D statically shaped polynomial buffer with i64 element type.
def CheddarGPU_PolyBuffer64 : CheddarGPU_StaticMemRefOfRank<[I64], 2,
"statically shaped 2D memref<L x N x i64> polynomial buffer">;

//===----------------------------------------------------------------------===//
// Dimension Helpers and PredOpTraits
//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -80,6 +88,18 @@ class CheddarGPU_EachCoversOutCount<string v, string out, int onlyRank = 0> : Pr
# !if(onlyRank, "m.getRank() != " # onlyRank # " || ", "")
# "m.getDimSize(0) >= " # CheddarGPU_OutCount<out>.result # "; })">>;

class CheddarGPU_DimAtLeast<string v, int d, int n> : PredOpTrait<
"dim " # d # " of '" # v # "' is >= " # n,
CPred<CheddarGPU_Dim<v, d>.result # " >= " # n>>;

class CheddarGPU_DimInRange<string v, int d, int lo, int hi> : PredOpTrait<
"dim " # d # " of '" # v # "' is in [" # lo # ", " # hi # "]",
CPred<CheddarGPU_Dim<v, d>.result # " >= " # lo # " && " # CheddarGPU_Dim<v, d>.result # " <= " # hi>>;

class CheddarGPU_LimbCountEqualsNumQPlusNumAux<string outBx> : PredOpTrait<
"dim 0 of '" # outBx # "' is num_q + num_aux",
CPred<"::llvm::cast<::mlir::MemRefType>((*$" # outBx # ".begin()).getType()).getDimSize(0) == static_cast<int64_t>($num_q + $num_aux)">>;

//===----------------------------------------------------------------------===//
// Variadic and Optional Operand Groups
//===----------------------------------------------------------------------===//
Expand All @@ -90,6 +110,10 @@ class CheddarGPU_SameCount<list<string> n>
class CheddarGPU_NonEmpty<string v>
: PredOpTrait<"'" # v # "' is non-empty", CPred<"!$" # v # ".empty()">>;

class CheddarGPU_CountInRange<string v, int lo, int hi> : PredOpTrait<
"'" # v # "' has " # lo # " to " # hi # " entries",
CPred<"static_cast<int64_t>($" # v # ".size()) >= " # lo # " && static_cast<int64_t>($" # v # ".size()) <= " # hi>>;

class CheddarGPU_EachTypeIs<string v, string ref> : PredOpTrait<
"every '" # v # "' has the type of '" # ref # "'",
CPred<"::llvm::all_of($" # v # ".getTypes(), [&](::mlir::Type t) { return t == $"
Expand All @@ -107,6 +131,19 @@ class CheddarGPU_EachSameRingAs<string v, string ref> : PredOpTrait<
# ref # ".getType()); return m.getDimSize(1) == r.getDimSize(1) && "
"m.getElementType() == r.getElementType(); })">>;

class CheddarGPU_EachDimAtLeast<string v, int d, string ref, int rd> : PredOpTrait<
"every '" # v # "' has dim " # d # " >= dim " # rd # " of '" # ref # "'",
CPred<"::llvm::all_of($" # v # ".getTypes(), [&](::mlir::Type t) { return "
"::llvm::cast<::mlir::MemRefType>(t).getDimSize(" # d # ") >= "
# CheddarGPU_Dim<ref, rd>.result # "; })">>;

class CheddarGPU_EvkDigits<string ref> : TraitList<[
CheddarGPU_NonEmpty<"evk_bx">,
CheddarGPU_SameCount<["evk_bx", "evk_ax"]>,
CheddarGPU_EachSameRingAs<"evk_bx", ref>,
CheddarGPU_EachSameRingAs<"evk_ax", ref>
]>;

class CheddarGPU_Getter<string n> { string call = "get" # snakeCaseToCamelCase<n>.ret # "()"; }

class CheddarGPU_BothOrNeither<string a, string b> : PredOpTrait<
Expand Down Expand Up @@ -198,6 +235,23 @@ def CheddarGPU_PairAttr : ConfinedAttr<DenseI64ArrayAttr, [DenseArrayCount<2>]>;
def CheddarGPU_InSplitAttr : ConfinedAttr<DenseI64ArrayAttr,
[DenseArrayCount<2>, DenseArrayNonNegative<DenseI64ArrayAttr>]>;

class CheddarGPU_IntMultipleOf<int n> : AttrConstraint<
CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() % " # n # " == 0">,
"whose value is a multiple of " # n>;

def CheddarGPU_PtWordOffsetsAttr : ConfinedAttr<DenseI64ArrayAttr, [
AttrConstraint<
CPred<"::llvm::all_of(::llvm::cast<::mlir::DenseI64ArrayAttr>($_self).asArrayRef(), [](int64_t v) { return v >= -1; })">,
"where all elements are >= -1">
]>;

def CheddarGPU_NumSlotsAttr : ConfinedAttr<I64Attr,
[IntPowerOf2, IntMinValue<256>, IntMaxValue<32768>]>;

class CheddarGPU_NumSlotsFitDegree<string poly> : PredOpTrait<
"num_slots <= degree / 2",
CPred<"static_cast<int64_t>($num_slots) <= " # CheddarGPU_Dim<poly, 1>.result # " / 2">>;

//===----------------------------------------------------------------------===//
// Native Traits
//===----------------------------------------------------------------------===//
Expand Down
13 changes: 13 additions & 0 deletions lib/Dialect/CheddarGPU/IR/CheddarGPUEnums.td
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,17 @@ def CheddarGPU_INTTEpilogueEnum : I32Enum<"INTTEpilogue", "INTT epilogue post-pr
let cppNamespace = "::mlir::heir::cheddar_gpu";
}

// === Hoist Phase Kind Enum ===

def CheddarGPU_HoistPhaseKind_CTS : I32EnumAttrCase<"CTS", 0, "cts">;
def CheddarGPU_HoistPhaseKind_STC : I32EnumAttrCase<"STC", 1, "stc">;

def CheddarGPU_HoistPhaseKindEnum : I32EnumAttr<"HoistPhaseKind", "Hoist phase kind (cts or stc)", [
CheddarGPU_HoistPhaseKind_CTS,
CheddarGPU_HoistPhaseKind_STC
]> {
let cppNamespace = "::mlir::heir::cheddar_gpu";
let genSpecializedAttr = 0;
}

#endif // LIB_DIALECT_CHEDDARGPU_IR_CHEDDARGPU_ENUMS_TD_
100 changes: 100 additions & 0 deletions lib/Dialect/CheddarGPU/IR/CheddarGPUOps.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,104 @@ LogicalResult INTTOp::verify() {
return success();
}

// === HoistBSFusedOp ===

LogicalResult HoistBSFusedOp::verify() {
size_t R = getOutBx().size();
size_t beta = getModUp().size();
auto firstOutBxType = ::llvm::cast<MemRefType>(getOutBx()[0].getType());
Type elemType = firstOutBxType.getElementType();

auto slabType = ::llvm::cast<MemRefType>(getEvkSlab().getType());

ArrayRef<int64_t> evkOffsets = getEvkByteOffsets();
if (static_cast<size_t>(evkOffsets.size()) != R &&
static_cast<size_t>(evkOffsets.size()) != R * beta * 2) {
return emitOpError("evk_byte_offsets length (")
<< evkOffsets.size() << ") must be either R (" << R
<< ") or R*beta*2 (" << R * beta * 2 << ")";
}
int64_t wordBytes = elemType.getIntOrFloatBitWidth() / 8;
for (size_t i = 0; i < evkOffsets.size(); ++i) {
int64_t off = evkOffsets[i];
if (off < 0 || off % wordBytes != 0) {
return emitOpError("evk_byte_offsets[")
<< i << "] (" << off << ") must be non-negative and a multiple of "
<< wordBytes;
}
if (off >= slabType.getDimSize(0)) {
return emitOpError("evk_byte_offsets[")
<< i << "] (" << off << ") exceeds evk_slab size ("
<< slabType.getDimSize(0) << ")";
}
}
return success();
}

// === HoistGSFusedOp ===

LogicalResult HoistGSFusedOp::verify() {
size_t G = getOutBx().size();
size_t B = getInBx().size();
auto firstOutBxType = ::llvm::cast<MemRefType>(getOutBx()[0].getType());
int64_t numLimbs = firstOutBxType.getDimSize(0);
int64_t ptLimbs = getPtLimbs();
if (ptLimbs < numLimbs) {
return emitOpError("pt_limbs (")
<< ptLimbs << ") must be >= ciphertext limbs (" << numLimbs << ")";
}

ArrayRef<int64_t> ptOffsets = getPtWordOffsets();
if (static_cast<size_t>(ptOffsets.size()) != G * B) {
return emitOpError("pt_word_offsets length (")
<< ptOffsets.size() << ") must equal G * B (" << G * B << ")";
}
return success();
}

// === HoistPhaseOp ===

LogicalResult HoistPhaseOp::verify() {
auto outBxType = ::llvm::cast<MemRefType>(getOutBx().getType());
auto inBxType = ::llvm::cast<MemRefType>(getInBx().getType());

HoistPhaseKind kind = getKind();
int64_t phase = getPhase();
int64_t expectedInLimbs = 0;
int64_t expectedOutLimbs = 0;
switch (kind) {
case HoistPhaseKind::CTS:
if (phase < 0 || phase > 3) {
return emitOpError("phase for 'cts' must be in [0, 3], but got ")
<< phase;
}
expectedInLimbs = 26 - phase;
expectedOutLimbs = 25 - phase;
break;
case HoistPhaseKind::STC:
if (phase < 0 || phase > 2) {
return emitOpError("phase for 'stc' must be in [0, 2], but got ")
<< phase;
}
expectedInLimbs = 14 - phase;
expectedOutLimbs = 13 - phase;
break;
}

int64_t inLimbs = inBxType.getDimSize(0);
int64_t outLimbs = outBxType.getDimSize(0);
if (inLimbs != expectedInLimbs) {
return emitOpError("input limb count for ")
<< stringifyHoistPhaseKind(kind) << " phase " << phase << " must be "
<< expectedInLimbs << ", but got " << inLimbs;
}
if (outLimbs != expectedOutLimbs) {
return emitOpError("output limb count for ")
<< stringifyHoistPhaseKind(kind) << " phase " << phase << " must be "
<< expectedOutLimbs << ", but got " << outLimbs;
}

return success();
}

} // namespace mlir::heir::cheddar_gpu
Loading
Loading