Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6fb328c
Read tensor.extract at affine indices as linalg.generic inputs
AlexanderViand Oct 27, 2026
069b2b3
Lay out reshapes that split or merge dimensions
AlexanderViand Oct 27, 2026
d741874
Lay out partial insert_slices into their destination
AlexanderViand Oct 27, 2026
42e922a
Add a layout-driven kernel for structured ops
AlexanderViand Oct 27, 2026
1039e1f
Lower structured ops without a dedicated kernel generically
AlexanderViand Oct 27, 2026
a08ae88
Generalize secret structured ops without a kernel in layout propagation
AlexanderViand Oct 27, 2026
5b37ca0
Add a Lattigo test of a padded depthwise convolution
AlexanderViand Oct 27, 2026
65347d5
Do not read the identity init of idempotent reductions
AlexanderViand Oct 27, 2026
3235be2
Fall back to the generic kernel when a dedicated layout rule does not…
AlexanderViand Oct 28, 2026
d94bce8
Erase unused results of linalg ops before layout propagation
AlexanderViand Oct 28, 2026
3f16759
Scan relations with an isl AST in enumeratePoints and lift locals in …
AlexanderViand Nov 13, 2026
4e2f0f0
Lay out reshapes by composing with the reshape index map
AlexanderViand Nov 17, 2026
1902a78
Check layout hoisting preconditions
AlexanderViand Nov 17, 2026
1696683
Skip unused inputs of generic ops in layout propagation and lowering
AlexanderViand Nov 17, 2026
4bd4a2a
Compose layouts in isl
AlexanderViand Nov 17, 2026
3c6051b
Scan relations along a reduced basis
AlexanderViand Nov 18, 2026
4bb5d87
Take induced lexmins over first replicas
AlexanderViand Nov 18, 2026
b6603a2
Pack folded layout chains through the composed layout
AlexanderViand Nov 18, 2026
db6b9e7
Pack folded layouts while scanning their points
AlexanderViand Nov 18, 2026
9514096
Keep the data layout on converted tensor slices
AlexanderViand Nov 18, 2026
80eec6d
Fold layouts without buffering their points
AlexanderViand Nov 18, 2026
352371b
Give strided generic inputs a polyphase layout
AlexanderViand Nov 19, 2026
e17b34b
Sum replicated generic results with a rotate-and-reduce
AlexanderViand Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions benchmark/isl/enumerate_points_benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,20 @@ static void BM_EnumeratePoints_LolaLayout12(benchmark::State& state) {
}
BENCHMARK(BM_EnumeratePoints_LolaLayout12)->Unit(benchmark::kSecond);

static void BM_EnumeratePoints_LenetConvFilterExpansion(
benchmark::State& state) {
BM_EnumeratePoints_helper(state, kLenetConvFilterExpansion);
}
BENCHMARK(BM_EnumeratePoints_LenetConvFilterExpansion)
->Unit(benchmark::kSecond);

static void BM_EnumeratePoints_LenetConvFilterDiagonals(
benchmark::State& state) {
BM_EnumeratePoints_helper(state, kLenetConvFilterDiagonals);
}
BENCHMARK(BM_EnumeratePoints_LenetConvFilterDiagonals)
->Unit(benchmark::kSecond);

} // namespace heir
} // namespace mlir

Expand Down
5 changes: 5 additions & 0 deletions benchmark/isl/get_ct_complement_benchmark.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ static void BM_GetCtComplementPoints_LayoutPooling1(benchmark::State& state) {
}
BENCHMARK(BM_GetCtComplementPoints_LayoutPooling1)->Unit(benchmark::kSecond);

static void BM_GetCtComplementPoints_CryptoNetsFilter(benchmark::State& state) {
BM_GetCtComplementPoints_helper(state, kCryptoNetsFilterRelation, 1024);
}
BENCHMARK(BM_GetCtComplementPoints_CryptoNetsFilter)->Unit(benchmark::kSecond);

// Earlier ways of finding which cts a layout uses, kept for the isl calls they
// make.

Expand Down
36 changes: 36 additions & 0 deletions benchmark/isl/relations.h
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,42 @@ inline constexpr char kCriteoLayout24[] =
"512 = 0 and 0 <= i0 <= 511 and 0 <= i1 <= 431 and 0 <= ct <= 511 and 0 <= "
"slot <= 32767 }";

// --- CryptoNets layouts ---

// Filter layout of a CryptoNets convolution, packed into 1024 ciphertexts.
// Some of its locals are not floor divisions of the other variables, and
// isl_set_foreach_point on the image of a single filter element did not finish
// within 30 minutes.
inline constexpr char kCryptoNetsFilterRelation[] =
"{ [i0, i1, i2, i3] -> [o0, o1] : exists (e1, e2, e3, e4, e5, e6: i1 "
"= 0 and (1 + i0 + i3 + o0) mod 2 = 0 and (i0 + o1) mod 2 = 0 and 0 "
"<= i0 <= 7 and 0 <= i2 <= 4 and 0 <= i3 <= 4 and 0 <= o0 <= 1023 and "
"0 <= o1 <= 8191 and 0 <= e1 <= 12 and 2e1 > -i2 and 4e2 > -i3 + o0 + "
"o1 and 2 - 2i3 + o0 + o1 <= 4e2 <= 29 - 2i3 + o0 + o1 and 2e5 >= -1 "
"+ i0 + i3 - o0 - o1 + 4e2 and 2e5 >= -50 - i0 - o1 + 4e3 and -2i0 - "
"o1 + 4*floor((i0)/4) - 4e1 + 4e3 <= 2e5 <= 1 - 2i0 - o1 + "
"4*floor((i0)/4) - 4e1 + 4e3 and 2e5 <= -i0 - o1 + 4e3 and 2e5 <= 25 "
"+ i0 and 2e5 <= i0 + i3 - o0 - o1 + 4e2 and 18 - 419i0 + 188i2 - "
"206i3 + 206o0 - 212o1 + 376e1 - 412e2 + 236e3 + 1024e4 <= 424e5 <= "
"21 - 419i0 + 188i2 - 206i3 + 206o0 - 212o1 + 376e1 - 412e2 + 236e3 + "
"1024e4 and -419i0 - 418o1 - 788e3 - 424e5 <= 2048e6 <= 3 - 419i0 - "
"418o1 - 788e3 - 424e5) }";

// --- LeNet constant packing ---

// Relations that tensor_ext.assign_layout enumerates to fold a constant weight
// of tests/Examples/openfhe/ckks/lenet into its packed form (min-slot-count
// 1024). They have 14.5 and 12.8 million points.
inline constexpr char kLenetConvFilterExpansion[] =
"{ [i0, i1, i2, i3, i4] -> [o0, o1, o2] : o0 = 576i0 + 24i2 + i3 and o1 = "
"i1 and o2 = i4 and 0 <= i0 <= 15 and 0 <= i1 <= 1 and 0 <= i2 <= 23 and "
"0 <= i3 <= 23 and 0 <= i4 <= 783 }";

inline constexpr char kLenetConvFilterDiagonals[] =
"{ [i0, i1] -> [o0, o1] : (i0 - i1 + o0) mod 2048 = 0 and (-i0 + o1) mod "
"16384 = 0 and 0 <= i0 <= 9215 and 0 <= i1 <= 1567 and 0 <= o0 <= 2047 "
"and 0 <= o1 <= 8191 }";

// --- Lola layouts ---

inline constexpr char kLolaLayout2[] =
Expand Down
3 changes: 3 additions & 0 deletions lib/Analysis/LayoutFoldingAnalysis/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,12 @@ cc_library(
srcs = ["LayoutFoldingAnalysis.cpp"],
hdrs = ["LayoutFoldingAnalysis.h"],
deps = [
"@heir//lib/Dialect/TensorExt/IR:Dialect",
"@heir//lib/Interface:HoistingInterfaces",
"@llvm-project//llvm:Support",
"@llvm-project//mlir:Analysis",
"@llvm-project//mlir:IR",
"@llvm-project//mlir:LinalgInterfaces",
"@llvm-project//mlir:Support",
],
)
40 changes: 15 additions & 25 deletions lib/Analysis/LayoutFoldingAnalysis/LayoutFoldingAnalysis.cpp
Original file line number Diff line number Diff line change
@@ -1,33 +1,22 @@
#include "lib/Analysis/LayoutFoldingAnalysis/LayoutFoldingAnalysis.h"

#include <set>

#include "lib/Dialect/TensorExt/IR/TensorExtOps.h"
#include "lib/Interface/HoistingInterfaces.h"
#include "llvm/include/llvm/Support/Debug.h" // from @llvm-project
#include "mlir/include/mlir/Analysis/DataFlowFramework.h" // from @llvm-project
#include "mlir/include/mlir/IR/BuiltinAttributes.h" // from @llvm-project
#include "mlir/include/mlir/IR/Operation.h" // from @llvm-project
#include "mlir/include/mlir/IR/Value.h" // from @llvm-project
#include "mlir/include/mlir/IR/Visitors.h" // from @llvm-project
#include "mlir/include/mlir/Interfaces/CallInterfaces.h" // from @llvm-project
#include "mlir/include/mlir/Support/LLVM.h" // from @llvm-project
#include "mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.h" // from @llvm-project
#include "mlir/include/mlir/IR/BuiltinAttributes.h" // from @llvm-project
#include "mlir/include/mlir/IR/Operation.h" // from @llvm-project
#include "mlir/include/mlir/IR/Value.h" // from @llvm-project
#include "mlir/include/mlir/IR/Visitors.h" // from @llvm-project
#include "mlir/include/mlir/Interfaces/CallInterfaces.h" // from @llvm-project
#include "mlir/include/mlir/Support/LLVM.h" // from @llvm-project

#define DEBUG_TYPE "layout-folding-analysis"

namespace mlir {
namespace heir {

namespace {

const std::set<std::string> layoutChangingOps = {
"linalg.reduce",
};

const std::set<std::string> layoutFixingOps = {
"tensor_ext.assign_layout",
};

} // namespace

void LayoutIsFreeAnalysis::setToEntryState(LayoutIsFreeLattice* lattice) {
// TODO(#1311): if the anchor is an arg of a non-main func, it may not be
// free.
Expand All @@ -36,8 +25,7 @@ void LayoutIsFreeAnalysis::setToEntryState(LayoutIsFreeLattice* lattice) {

LogicalResult isKnownToBeFree(Operation* operation,
ArrayRef<const LayoutIsFreeLattice*> operands) {
if (operands.empty() ||
layoutFixingOps.contains(operation->getName().getStringRef().str())) {
if (operands.empty() || isa<tensor_ext::AssignLayoutOp>(operation)) {
LLVM_DEBUG(llvm::dbgs()
<< "Op " << operation->getName() << " is known to be free.\n");
return success();
Expand Down Expand Up @@ -73,9 +61,11 @@ LogicalResult LayoutIsFreeAnalysis::visitOperation(
if (!resultLayoutIsFree.getValue()) break;
}

// Distinguished ops that change the layout of their results.
std::string opName(operation->getName().getStringRef());
if (layoutChangingOps.contains(opName)) {
// Operations that cannot hoist a layout conversion through their results for
// free fix the layout of their results.
if (isa<linalg::LinalgOp>(operation) ||
!isa<tensor_ext::ConvertLayoutOp, LayoutConversionHoistableOpInterface>(
operation)) {
LLVM_DEBUG(llvm::dbgs() << "Op is designated to make results not free\n");
resultLayoutIsFree = LayoutIsFree(false);
}
Expand Down
23 changes: 16 additions & 7 deletions lib/Dialect/TensorExt/IR/TensorExtAttributes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,23 @@ LayoutAttr LayoutAttr::getFromIntegerRelation(::mlir::MLIRContext* context,

LayoutAttr LayoutAttr::composeLayouts(::mlir::ArrayAttr arrayAttr,
::mlir::MLIRContext* context) {
auto layoutAttrs = arrayAttr.getAsRange<LayoutAttr>();
auto it = layoutAttrs.begin();
IntegerRelation composedRel = (*it).getIntegerRelation();
++it;
for (; it != layoutAttrs.end(); ++it) {
composedRel.compose((*it).getIntegerRelation());
// Compose in isl, which keeps the definitions of the divisions in each
// layout, so the intermediate coordinates are eliminated through them.
isl_ctx* ctx = isl_ctx_alloc();
isl_basic_map* composed = nullptr;
for (LayoutAttr layout : arrayAttr.getAsRange<LayoutAttr>()) {
isl_basic_map* next =
isl_basic_map_read_from_str(ctx, layout.getLayoutStr().c_str());
composed = composed ? isl_basic_map_apply_range(composed, next) : next;
}
return LayoutAttr::getFromIntegerRelation(context, composedRel);
composed = isl_basic_map_set_dim_name(composed, isl_dim_out, 0, "ct");
composed = isl_basic_map_set_dim_name(composed, isl_dim_out, 1, "slot");
char* resultStr = isl_basic_map_to_str(composed);
std::string layoutStr(resultStr);
free(resultStr);
isl_basic_map_free(composed);
isl_ctx_free(ctx);
return LayoutAttr::get(context, layoutStr);
}

} // namespace tensor_ext
Expand Down
35 changes: 35 additions & 0 deletions lib/Kernel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,19 @@ cc_library(
],
)

cc_library(
name = "GenericKernelImplementation",
hdrs = ["GenericKernelImplementation.h"],
deps = [
":AbstractValue",
":ArithmeticDag",
":KernelImplementation",
"@heir//lib/Utils/Layout:GenericKernel",
"@llvm-project//llvm:Support",
"@llvm-project//mlir:Support",
],
)

cc_library(
name = "Utils",
srcs = ["Utils.cpp"],
Expand Down Expand Up @@ -192,6 +205,28 @@ cc_test(
],
)

cc_test(
name = "GenericKernelFuzzTest",
srcs = ["GenericKernelFuzzTest.cpp"],
deps = [
":AbstractValue",
":ArithmeticDag",
":EvalVisitor",
":GenericKernelImplementation",
"@fuzztest//fuzztest",
"@fuzztest//fuzztest:fuzztest_gtest_main",
"@heir//lib/Utils/Layout:Evaluate",
"@heir//lib/Utils/Layout:GenericKernel",
"@heir//lib/Utils/Layout:IslConversion",
"@heir//lib/Utils/Layout:Utils",
"@llvm-project//llvm:Support",
"@llvm-project//mlir:Analysis",
"@llvm-project//mlir:AsmParser",
"@llvm-project//mlir:IR",
"@llvm-project//mlir:Support",
],
)

cc_test(
name = "MatvecZeroDiagonalsFuzzTest",
srcs = ["MatvecZeroDiagonalsFuzzTest.cpp"],
Expand Down
Loading
Loading