From df37c4c9f4a1225304b024040a58074f151fdd5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Sat, 10 Oct 2026 14:51:28 +0100 Subject: [PATCH 01/10] Report RTS stats in ghcide-bench --- ghcide-bench/src/Experiments.hs | 11 +++++++++++ ghcide-bench/src/Experiments/Types.hs | 1 + 2 files changed, 12 insertions(+) diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index 6759128747..5d32c3cdc0 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -419,6 +419,7 @@ configP = ) <*> optional (strOption (long "shake-profiling" <> metavar "PATH")) <*> optional (strOption (long "ot-profiling" <> metavar "DIR" <> help "Enable OpenTelemetry and write eventlog for each benchmark in DIR")) + <*> optional (strOption (long "rts-profiling" <> metavar "DIR" <> help "Enable RTS profiling and write stats for each benchmark in DIR")) <*> strOption (long "csv" <> metavar "PATH" <> value "results.csv" <> showDefault) <*> flag Cabal Stack (long "stack" <> help "Use stack (by default cabal is used)") <*> many (strOption (long "ghcide-options" <> help "additional options for ghcide")) @@ -495,6 +496,8 @@ runBenchmarksFun dir allBenchmarks = do whenJust (otMemoryProfiling ?config) $ \eventlogDir -> createDirectoryIfMissing True eventlogDir + whenJust (rtsProfiling ?config) $ \rtsDir -> + createDirectoryIfMissing True rtsDir lspConfig <- if Experiments.Types.lspConfig ?config then either error id . eitherDecodeStrict' <$> BS.getContents @@ -547,6 +550,14 @@ runBenchmarksFun dir allBenchmarks = do ] | Just dir <- [otMemoryProfiling ?config] ] + ++ concat + [ [ "+RTS" + , "-t" ++ (dir map (\c -> if c == ' ' then '-' else c) name <.> "rts") + , "--machine-readable" + , "-RTS" + ] + | Just dir <- [rtsProfiling ?config] + ] ++ ghcideOptions ?config ++ concat [ ["--shake-profiling", path] | Just path <- [shakeProfiling ?config] diff --git a/ghcide-bench/src/Experiments/Types.hs b/ghcide-bench/src/Experiments/Types.hs index db33744912..fd947e8cbc 100644 --- a/ghcide-bench/src/Experiments/Types.hs +++ b/ghcide-bench/src/Experiments/Types.hs @@ -22,6 +22,7 @@ data Config = Config -- For some reason, the Shake profile files are truncated and won't load shakeProfiling :: !(Maybe FilePath), otMemoryProfiling :: !(Maybe FilePath), + rtsProfiling :: !(Maybe FilePath), outputCSV :: !FilePath, buildTool :: !CabalStack, ghcideOptions :: ![String], From abb264f7ac01dc3add7a99ffe64583f63a51798a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Sat, 10 Oct 2026 23:09:55 +0100 Subject: [PATCH 02/10] Make ghcide-bench produce real csv --- ghcide-bench/src/Experiments.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index 5d32c3cdc0..ef32e5f919 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -526,7 +526,7 @@ runBenchmarksFun dir allBenchmarks = do let headers = map fst summaryColumns summaryRow showTime b r = [ renderCell showTime (cell b r) | (_, cell) <- summaryColumns ] - csv = T.unlines $ map (T.intercalate ", ") + csv = T.unlines $ map (T.intercalate ",") (headers : [ summaryRow showMs b r | (b, r) <- results ]) TIO.writeFile (outputCSV ?config) csv From 011ba2a44f5127645a3e969890632196838a9293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Sun, 11 Oct 2026 11:47:19 +0100 Subject: [PATCH 03/10] [CI] Add scipt to merge benchmark data --- .github/bench.hs | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/bench.hs diff --git a/.github/bench.hs b/.github/bench.hs new file mode 100644 index 0000000000..9dfb424090 --- /dev/null +++ b/.github/bench.hs @@ -0,0 +1,73 @@ +#!/usr/bin/env cabal +{- cabal: +build-depends: base, split, text, cassava, vector, bytestring, directory, filepath, containers, extra +-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE OverloadedStrings #-} +module Main where + +import Control.Monad (forM_) +import qualified Data.ByteString.Lazy +import Data.Csv (FromNamedRecord (..), Header, + decodeByName, decodeByNameWith, (.:)) +import Data.List.Extra (stripSuffix) +import Data.Map (elems, intersectionWith, toList) +import qualified Data.Map +import Data.Maybe (fromJust) +import qualified Data.Text as T +import qualified Data.Text.IO +import qualified Data.Vector (Vector, toList) +import System.Directory (listDirectory) +import System.Environment (getArgs) +import System.FilePath (()) + +data CsvRecord = CsvRecord { name:: !String, success :: !T.Text, firstTime :: !Double, avgTime :: !Double} + deriving Show +instance FromNamedRecord CsvRecord where + parseNamedRecord m = CsvRecord <$> + m .: "name" <*> + m .: "success" <*> + m .: "1stBuildT" <*> + m .: "avgPerRespT" + + +readCsv :: FilePath -> IO ([CsvRecord]) +readCsv fp= do + contents <- Data.ByteString.Lazy.readFile fp + let Right csv = decodeByName contents :: Either String (Header,Data.Vector.Vector CsvRecord ) + pure $ Data.Vector.toList $ snd csv + +readRts :: FilePath ->IO [(String,Double)] +-- right now we don't worry about int vs double metrics +readRts fp= do + contents <- Data.Text.IO.readFile fp + let contents' = T.unlines $ tail $ T.lines contents + let contents'' = read $ T.unpack contents' ::[(String,String)] + pure$ map (\(x,y) -> (x,read y)) contents'' +readAllRts :: FilePath -> IO [(FilePath,[(String,Double)])] +readAllRts fp = do + files <- listDirectory fp + mapM (\x -> readRts (fp x) >>= \y -> pure (normalizeName x,y)) files + where + normalizeName :: String -> String + normalizeName s = fromJust $ stripSuffix ".rts" s + +main :: IO () +main = do + (csvPath:rtsDir:_) <- getArgs + csv <- readCsv csvPath + let csvMap = Data.Map.fromList $ map (\x -> (map (\c -> if c == ' ' then '-' else c) $ name x,x)) csv + rtsMap<- Data.Map.fromList `fmap` readAllRts rtsDir + let measurements = intersectionWith (\x -> \y -> (x,y)) csvMap rtsMap + + forM_ (elems measurements) printResult +printResult :: (CsvRecord,[(String,Double)]) -> IO () +printResult (csv,xs) = do + putStrLn $ name csv + putStrLn $ "\tSuccess: " ++ ( T.unpack $ success csv) + if (T.unpack $ success csv) == "True" then do + putStrLn $ "\t1st time: " ++ ( show $ firstTime csv) + putStrLn $ "\tavg time: " ++ ( show $ avgTime csv) + forM_ xs (\ (n,m) -> putStrLn $ "\t"++n ++ ": "++ (show m)) + else pure () + From 57dfd0eaf63b7d2fd516533b3f46d95934b3004f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Sun, 11 Oct 2026 12:01:32 +0100 Subject: [PATCH 04/10] Change benchmarking workflow to use ghcide-bench --- .github/bench.hs | 0 .github/workflows/bench.yml | 124 +++++++----------------------------- 2 files changed, 23 insertions(+), 101 deletions(-) mode change 100644 => 100755 .github/bench.hs diff --git a/.github/bench.hs b/.github/bench.hs old mode 100644 new mode 100755 diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c794fffd4c..9abbb24ccd 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -50,7 +50,7 @@ jobs: , ".gitlab/**" ]' - bench_init: + bench: if: needs.pre_job.outputs.should_skip != 'true' needs: pre_job runs-on: ${{ matrix.os }} @@ -76,6 +76,17 @@ jobs: # not it is a merge commit, so this is how we do that. ref: "${{ github.event.pull_request.head.sha }}" + - name: Restore cache + id: cache-restore + uses: actions/cache/restore@v6 + with: + path: | + dist-newstyle + bench/example + ~/.cabal + key: ${{ runner.os }}-${{ matrix.ghc }}-bench + + - run: git fetch origin master # check the master branch for benchmarking - uses: ./.github/actions/setup-build @@ -89,110 +100,21 @@ jobs: - run: cabal configure --enable-benchmarks --max-backjumps 12000 - name: Build - run: cabal build haskell-language-server:benchmark - - - name: Bench init - run: cabal bench -j --benchmark-options="all-binaries" - - # tar is required to preserve file permissions - # compression speeds up upload/download nicely - - name: tar workspace - run: tar -czf workspace.tar.gz * .git - - - name: tar cabal - run: | - cd ~/.cabal - tar -czf cabal.tar.gz * - - - name: Upload workspace - uses: actions/upload-artifact@v7 - with: - name: workspace-${{ matrix.ghc }}-${{ matrix.os }} - retention-days: 1 - path: workspace.tar.gz - - - name: Upload .cabal - uses: actions/upload-artifact@v7 - with: - name: cabal-home-${{ matrix.ghc }}-${{ matrix.os }} - retention-days: 1 - path: ~/.cabal/cabal.tar.gz - - bench_example: - if: contains(github.event.pull_request.labels.*.name, 'performance') - needs: [bench_init, pre_job] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }} - os: [ubuntu-latest] - cabal: ['3.16'] - example: ['cabal', 'lsp-types'] - - steps: - - uses: haskell-actions/setup@v2.11 - with: - ghc-version : ${{ matrix.ghc }} - cabal-version: ${{ matrix.cabal }} - enable-stack: false - - - name: Download cabal home - uses: actions/download-artifact@v8 - with: - name: cabal-home-${{ matrix.ghc }}-${{ matrix.os }} - path: . - - - name: Download workspace - uses: actions/download-artifact@v8 - with: - name: workspace-${{ matrix.ghc }}-${{ matrix.os }} - path: . - - - name: untar - run: | - mkdir -p ~/.cabal - tar xzf workspace.tar.gz - tar xzf cabal.tar.gz --directory ~/.cabal + run: cabal build exe:ghcide-bench - name: Bench - run: cabal bench -j --benchmark-options="${{ matrix.example }}" + run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv - name: Display results - run: | - column -s, -t < bench-results/unprofiled/${{ matrix.example }}/results.csv | tee bench-results/unprofiled/${{ matrix.example }}/results.txt - echo - echo "Performance Diff(comparing to its previous Version):" - column -s, -t < bench-results/unprofiled/${{ matrix.example }}/resultDiff.csv | tee bench-results/unprofiled/${{ matrix.example }}/resultDiff.txt - - - name: tar benchmarking artifacts - run: find bench-results -name "*.csv" | xargs tar -czf benchmark-artifacts.tar.gz + run: .github/bench.hs results.csv rts-stats - - name: Archive benchmarking artifacts - uses: actions/upload-artifact@v7 + - name: Save cache + id: cache-save + uses: actions/cache/save@v6 with: - name: bench-results-${{ matrix.example }}-${{ runner.os }}-${{ matrix.ghc }} - path: benchmark-artifacts.tar.gz + path: | + dist-newstyle + bench/example + ~/.cabal + key: ${{ runner.os }}-${{ matrix.ghc }}-bench - - name: tar benchmarking logs - # We dont' store the eventlogs because the CI workers risk running out of disk space - run: find bench-results -name "*.log" -or -name "*.hp" | xargs tar -czf benchmark-logs.tar.gz - - - name: Archive benchmark logs - uses: actions/upload-artifact@v7 - with: - name: bench-logs-${{ matrix.example }}-${{ runner.os }}-${{ matrix.ghc }} - path: benchmark-logs.tar.gz - - bench_post_job: - if: always() - runs-on: ubuntu-latest - needs: [pre_job, bench_init, bench_example] - steps: - - run: | - echo "jobs info: ${{ toJSON(needs) }}" - - if: contains(needs.*.result, 'failure') - run: exit 1 - - if: contains(needs.*.result, 'cancelled') && needs.pre_job.outputs.should_skip != 'true' - run: exit 1 From c0f3c7541c46321610796639bd20292713ef5e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 22:20:24 +0100 Subject: [PATCH 05/10] [CI] Skip extra cache steps in bench --- .github/workflows/bench.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9abbb24ccd..3a158ed8f6 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -76,17 +76,6 @@ jobs: # not it is a merge commit, so this is how we do that. ref: "${{ github.event.pull_request.head.sha }}" - - name: Restore cache - id: cache-restore - uses: actions/cache/restore@v6 - with: - path: | - dist-newstyle - bench/example - ~/.cabal - key: ${{ runner.os }}-${{ matrix.ghc }}-bench - - - run: git fetch origin master # check the master branch for benchmarking - uses: ./.github/actions/setup-build @@ -108,13 +97,4 @@ jobs: - name: Display results run: .github/bench.hs results.csv rts-stats - - name: Save cache - id: cache-save - uses: actions/cache/save@v6 - with: - path: | - dist-newstyle - bench/example - ~/.cabal - key: ${{ runner.os }}-${{ matrix.ghc }}-bench From 58aeb03babbfa3a43d319816f79361dfb80fef1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 22:21:16 +0100 Subject: [PATCH 06/10] [CI] Dont build benchmark target --- ghcide-bench/src/Experiments.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index ef32e5f919..796f831efe 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -755,9 +755,9 @@ setup = do return path checkExampleModulesExist benchDir (example ?config) - case (buildTool ?config, exampleDetails (example ?config)) of - (Cabal, ExampleHackage{}) -> buildCabalExample benchDir - _ -> return () + -- case (buildTool ?config, exampleDetails (example ?config)) of + -- (Cabal, ExampleHackage{}) -> buildCabalExample benchDir + -- _ -> return () whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True From 7b9a8f0451f2bfd282c7c3bcf557edfb711c651e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 22:43:35 +0100 Subject: [PATCH 07/10] [CI] Build hls in benchmark --- .github/workflows/bench.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 3a158ed8f6..29850dc1b9 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -88,13 +88,27 @@ jobs: # for dependency resolution failure - run: cabal configure --enable-benchmarks --max-backjumps 12000 + - name: Build hls + run: cabal build exe:haskell-language-server + - name: Build run: cabal build exe:ghcide-bench - name: Bench - run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv + run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv --ghcide $(cabal list-bin exe:haskell-language-server) - name: Display results run: .github/bench.hs results.csv rts-stats + bench_post_job: + if: always() + runs-on: ubuntu-latest + needs: [pre_job, bench] + steps: + - run: | + echo "jobs info: ${{ toJSON(needs) }}" + - if: contains(needs.*.result, 'failure') + run: exit 1 + - if: contains(needs.*.result, 'cancelled') && needs.pre_job.outputs.should_skip != 'true' + run: exit 1 From f38c4cd57206c34df111945923ed74479c941790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 23:10:01 +0100 Subject: [PATCH 08/10] Revert "[CI] Dont build benchmark target" This reverts commit 58aeb03babbfa3a43d319816f79361dfb80fef1e. --- ghcide-bench/src/Experiments.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index 796f831efe..ef32e5f919 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -755,9 +755,9 @@ setup = do return path checkExampleModulesExist benchDir (example ?config) - -- case (buildTool ?config, exampleDetails (example ?config)) of - -- (Cabal, ExampleHackage{}) -> buildCabalExample benchDir - -- _ -> return () + case (buildTool ?config, exampleDetails (example ?config)) of + (Cabal, ExampleHackage{}) -> buildCabalExample benchDir + _ -> return () whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True From bae11d732c4e1e4bbd76259143bc9d9e96ab8c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 23:10:40 +0100 Subject: [PATCH 09/10] [CI] Enable verbose in ghcide-bench --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 29850dc1b9..c02cbbe5f2 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -95,7 +95,7 @@ jobs: run: cabal build exe:ghcide-bench - name: Bench - run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv --ghcide $(cabal list-bin exe:haskell-language-server) + run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv --ghcide $(cabal list-bin exe:haskell-language-server) --verbose - name: Display results run: .github/bench.hs results.csv rts-stats From 80938a890c41da3ee9e070487817c6998ea231ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dybiec?= Date: Fri, 23 Oct 2026 23:24:48 +0100 Subject: [PATCH 10/10] Revert "[CI] Enable verbose in ghcide-bench" This reverts commit bae11d732c4e1e4bbd76259143bc9d9e96ab8c95. --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c02cbbe5f2..29850dc1b9 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -95,7 +95,7 @@ jobs: run: cabal build exe:ghcide-bench - name: Bench - run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv --ghcide $(cabal list-bin exe:haskell-language-server) --verbose + run: cabal run ghcide-bench -- --example-name cabal --samples 10 --no-clean --rts-profiling rts-stats --csv results.csv --ghcide $(cabal list-bin exe:haskell-language-server) - name: Display results run: .github/bench.hs results.csv rts-stats