From 67acd361d8e908aa015ef92e2c85ff62b8cab919 Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:09:09 +0200 Subject: [PATCH 1/4] Split the pipeline into .CI/common.groovy and .CI/report.groovy .CI/Jenkinsfile holds the stages; what they do is a function in .CI/common.groovy, or in .CI/report.groovy for the pages published from the results. A setup stage loads both and the stages reach them as common.() and report.(), the split the OpenModelica repository uses for its own pipeline. Loading them is a stage of its own because every stage that tests a branch skips the checkout that would otherwise bring the files into its workspace. The report stage shrank the most: report.py followed by `mv overview.html ` was written out twenty times and is an overview() call now, and the four pages that -oldinst, -fmi, -dae and -newbackend-dae each produce are one overviewSet(). It runs the same commands in a different order, which no page depends on. Two things changed while moving: - The locals of runRegressiontest are declared with def. Without it they land in the binding the whole pipeline shares, where the stages running in parallel overwrite each other's. - The FMI jobs of the maintenance branches build OMSimulator from origin/master rather than master. git fetch does not update a local branch, so resetting to one kept whatever commit another job had left in the shared workspace. All six jobs take the ref from one variable now. A new workflow lints every Jenkinsfile and *.groovy with npm-groovy-lint and the rules in .groovylintrc.json: a syntax error is reported as an error and anything the rules complain about as a warning, both of which fail the job, so a Jenkinsfile that does not parse is found before a build runs it. Co-Authored-By: Claude Opus 5 --- .CI/Jenkinsfile | 1363 ++++++++--------------------- .CI/common.groovy | 574 ++++++++++++ .CI/report.groovy | 157 ++++ .github/workflows/lint-groovy.yml | 37 + .gitignore | 1 + .groovylintrc.json | 32 + README.md | 10 + 7 files changed, 1186 insertions(+), 988 deletions(-) create mode 100644 .CI/common.groovy create mode 100644 .CI/report.groovy create mode 100644 .github/workflows/lint-groovy.yml create mode 100644 .groovylintrc.json diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index a3a379b..ced4bb1 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -1,3 +1,9 @@ +def common +def report + +// The OMSimulator the FMI jobs build and simulate the FMUs with. +def omsimulatorRef = 'origin/master' + pipeline { agent none parameters { @@ -52,425 +58,426 @@ pipeline { LIBTEST_DB = 'postgresql://om@openmodelica.org/omdb' } stages { - stage('test') { parallel { - - stage('v1.26') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26', '', '', false, '', '', false, false) + // Everything the stages call lives in .CI/common.groovy and .CI/report.groovy; + // this is where they are loaded, since the stages testing a branch skip the + // checkout that would bring the files into their workspace. + stage('setup') { + agent { + label 'linux' + } + steps { + script { + common = load("${env.WORKSPACE}/.CI/common.groovy") + report = load("${env.WORKSPACE}/.CI/report.groovy") } } - - stage('v1.27') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + stage('test') { + parallel { + stage('v1.26') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.v1_26 } + } + steps { + script { common.runRegressiontest('maintenance/v1.26', 'v1.26', '', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27', '', '', false, '', '', false, false) - } - } - stage('master') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('v1.27') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.v1_27 } + } + steps { + script { common.runRegressiontest('maintenance/v1.27', 'v1.27', '', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.master } - } - steps { - runRegressiontest('master', 'master', '', '', false, '', '', false, false) - } - } - stage('conversion script') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('master') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.master } + } + steps { + script { common.runRegressiontest('master', 'master', '', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.conversion_script } - } - steps { - runRegressiontest('master', 'conversion', '', '', false, '', '', false, true) - } - } - stage('pull request') { - agent { - node { - label "${params.pull_request_node ?: 'ryzen-5950x-1'}" - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('conversion script') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.conversion_script } + } + steps { + script { common.runRegressiontest('master', 'conversion', '', '', false, '', '', false, true) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { pullRequest() } - } - steps { - script { - if (!(pullRequest() ==~ /[0-9]+/)) { - error "pull_request is a pull request number; got '${params.pull_request}'" + + stage('pull request') { + agent { + node { + label "${params.pull_request_node ?: 'ryzen-5950x-1'}" + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { common.pullRequest() } + } + steps { + script { + common.checkPullRequestNumber() + common.checkPullRequestExists() + common.removeStalePullRequestBuilds() + common.runRegressiontest("pr/${common.pullRequest()}", "pr/${common.pullRequest()}", '', '', false, '', '', false, false, 0, params.pull_request_config ?: 'configs/conf.json') } } - // Before the clone, the reset and the build: a number that is not a - // pull request costs minutes to find out about otherwise. Issues and - // pull requests share one numbering, so an issue number gets this far. - sh """ - if ! git ls-remote --exit-code https://github.com/OpenModelica/OpenModelica.git 'refs/pull/${pullRequest()}/*' > /dev/null; then - echo "OpenModelica/OpenModelica has no pull request ${pullRequest()}. Issues and pull requests share one numbering there, so check that ${pullRequest()} is not the number of an issue." - exit 1 - fi - """ - // One build of omc per pull request is kept, as for a branch, and they - // accumulate: a pull request is tested once and never again. The - // second line is the layout of the runs before they moved under pr/. - sh ''' - find "$HOME/saved_omc/pr" -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} ";" 2> /dev/null || true - find "$HOME/saved_omc" -mindepth 1 -maxdepth 1 -name "pr-*" -type d -mtime +14 -exec rm -rf {} ";" || true - ''' - runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", '', '', false, '', '', false, false, 0, params.pull_request_config ?: 'configs/conf.json') } - } - stage('newInst-newBackend') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('newInst-newBackend') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.newInst_newBackend } + } + steps { + script { common.runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.newInst_newBackend } - } - steps { - runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', false, '', '', false, false) - } - } - stage('v1.26 FMI') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('v1.26 FMI') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmi_v1_26 || params.fmpy_fmi_v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) - } - } - stage('v1.27 FMI') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.fmi_v1_26 || params.fmpy_fmi_v1_26 } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmi_v1_27 || params.fmpy_fmi_v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) - } - } - stage('master FMI') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + steps { + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmi_master || params.fmpy_fmi_master } + stage('v1.27 FMI') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.fmi_v1_27 || params.fmpy_fmi_v1_27 } + } + steps { + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) } + } } - steps { - runRegressiontest('master', 'master-fmi', '', 'origin/master', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) + stage('master FMI') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.fmi_master || params.fmpy_fmi_master } + } + steps { + script { common.runRegressiontest('master', 'master-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) } + } } - } - stage('v1.26 CVODE CS-FMUs with OMSimulator') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('v1.26 CVODE CS-FMUs with OMSimulator') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cs_fmu_cvode_v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), false, '', '', false, false) - } - } - stage('v1.27 CVODE CS-FMUs with OMSimulator') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cs_fmu_cvode_v1_26 } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cs_fmu_cvode_v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorHash(), false, '', '', false, false) - } - } - stage('master CVODE CS-FMUs with OMSimulator') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + steps { + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cs_fmu_cvode_master } + stage('v1.27 CVODE CS-FMUs with OMSimulator') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cs_fmu_cvode_v1_27 } + } + steps { + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } + } } - steps { - runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', 'origin/master', false, '', '', false, false) + stage('master CVODE CS-FMUs with OMSimulator') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cs_fmu_cvode_master } + } + steps { + script { common.runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } + } } - } - stage('newInst-daeMode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('newInst-daeMode') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.newInst_daeMode } - } - steps { - runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', false, '', '', false, false) - } - } - stage('newBackend-daeMode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.newInst_daeMode } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.newBackend_daeMode } - } - steps { - runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', false, '', '', false, false) - } - } - stage('oldInst') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + steps { + script { common.runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.oldInst } - } - steps { - runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', false, '', '', false, false) - } - } - stage('cvode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('newBackend-daeMode') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cvode } - } - steps { - runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s cvode', '', false, false) - } - } - stage('gbode') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.newBackend_daeMode } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.gbode } - } - steps { - runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s gbode -gbm=radauIIA3', '', false, false) - } - } - stage('ida') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + steps { + script { common.runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.ida } - } - steps { - runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s ida', '', false, false) - } - } - stage('wasm-jit') { - agent { - node { - label 'ryzen-9950x' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('oldInst') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.oldInst } + } + steps { + script { common.runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.wasm_jit } - } - steps { - runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, '', '--wasmjitrunner=sim,me,cs', false, false, 0, 'configs/conf.json', - '-DOM_OMC_ENABLE_RUST=ON -DRUST_OMC_CI=ON -DRUST_OMC_THREADS=4 -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache') - } - } - stage('generateSymbolicJacobian') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('cvode') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cvode } + } + steps { + script { common.runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s cvode', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.generateSymbolicJacobian } + stage('gbode') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.gbode } + } + steps { + script { common.runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s gbode -gbm=radauIIA3', '', false, false) } + } } - steps { - runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', false, '', '', false, false) + stage('ida') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.ida } + } + steps { + script { common.runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s ida', '', false, false) } + } } - } - stage('heavy_tests') { - agent { - node { - label 'ryzen-5950x-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('wasm-jit') { + agent { + node { + label 'ryzen-9950x' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.wasm_jit } + } + steps { + script { + common.runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, '', '--wasmjitrunner=sim,me,cs', false, false, 0, 'configs/conf.json', + '-DOM_OMC_ENABLE_RUST=ON -DRUST_OMC_CI=ON -DRUST_OMC_THREADS=4 -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache') + } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.heavy_tests } + stage('generateSymbolicJacobian') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.generateSymbolicJacobian } + } + steps { + script { common.runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', false, '', '', false, false) } + } } - steps { - runRegressiontest('master', 'heavy_tests', '', '', false, '', '', false, false, 1, 'configs/heavy_tests.json') + stage('heavy_tests') { + agent { + node { + label 'ryzen-5950x-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.heavy_tests } + } + steps { + script { common.runRegressiontest('master', 'heavy_tests', '', '', false, '', '', false, false, 1, 'configs/heavy_tests.json') } + } } - } - stage('C++ v1.26') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('C++ v1.26') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cpp_v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) - } - } - stage('C++ v1.27') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cpp_v1_26 } + } + steps { + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } } } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cpp_v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) + stage('C++ v1.27') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cpp_v1_27 } + } + steps { + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } + } } - } - stage('C++') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' + stage('C++') { + agent { + node { + label 'ryzen-5950x-2-1' + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.cpp } + } + steps { + script { common.runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.cpp } - } - steps { - runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } } - } } + } stage('report') { agent { dockerfile { @@ -482,7 +489,7 @@ pipeline { } when { beforeAgent true - expression { params.v1_25 || params.v1_26 || params.v1_27 || params.master || params.conversion_script || params.report_ryzen_5950x_1 || params.report_ryzen_5950x_2 || params.newInst_newBackend || params.generateSymbolicJacobian || params.heavy_tests || params.fmi_v1_25 || params.fmi_v1_26 || params.fmi_v1_27 || params.fmi_master || params.fmpy_fmi_v1_25 || params.fmpy_fmi_v1_26 || params.fmpy_fmi_v1_27 || params.fmpy_fmi_master || params.newInst_daeMode || params.newBackend_daeMode || params.oldInst || params.cpp || params.cvode || params.gbode || params.ida || params.wasm_jit} + expression { params.v1_25 || params.v1_26 || params.v1_27 || params.master || params.conversion_script || params.report_ryzen_5950x_1 || params.report_ryzen_5950x_2 || params.newInst_newBackend || params.generateSymbolicJacobian || params.heavy_tests || params.fmi_v1_25 || params.fmi_v1_26 || params.fmi_v1_27 || params.fmi_master || params.fmpy_fmi_v1_25 || params.fmpy_fmi_v1_26 || params.fmpy_fmi_v1_27 || params.fmpy_fmi_master || params.newInst_daeMode || params.newBackend_daeMode || params.oldInst || params.cpp || params.cvode || params.gbode || params.ida || params.wasm_jit } } environment { GITBRANCHES = 'maintenance/v1.20 maintenance/v1.21 maintenance/v1.22 maintenance/v1.23 maintenance/v1.24 maintenance/v1.25 maintenance/v1.26 maintenance/v1.27 master newInst-newBackend' @@ -502,94 +509,12 @@ pipeline { PGPASSFILE = credentials('omdb-pgpass') } steps { - sh 'rm -rf *.html history' - sh ''' - if ! test -d OpenModelica; then - git clone https://openmodelica.org/git-readonly/OpenModelica.git - fi - cd OpenModelica - git fetch - ''' - sh './clean-empty-omcversion-dates.py' - - sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} conversion heavy_tests generateSymbolicJacobian gbode cvode ida" - sh "./all-plots.py ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} conversion heavy_tests generateSymbolicJacobian gbode cvode ida" - - sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" - sh 'mv overview.html overview-combined.html' - sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf-old.json" - sh "mv overview.html overview-old-libs.html" - sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf-nonstandard.json" - sh "mv overview.html overview-nonstandard-libs.html" - sh "./report.py --branches='${env.GITBRANCHES_SPECIAL} conversion' configs/conf.json" - sh "mv overview.html overview-special-jobs.html" - - sh "./report.py --branches='generateSymbolicJacobian' configs/conf.json" - sh "mv overview.html overview-generateSymbolicJacobian.html" - - sh "./report.py --branches='heavy_tests' configs/heavy_tests.json" - sh "mv overview.html overview-heavy_tests.html" - - sh "./report.py --branches='${env.GITBRANCHES_NEWINST}' configs/conf.json" - sh "mv overview.html overview-oldinst.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWINST}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" - sh "mv overview.html overview-combined-oldinst.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWINST}' configs/conf-old.json" - sh "mv overview.html overview-old-libs-oldinst.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWINST}' configs/conf-nonstandard.json" - sh "mv overview.html overview-nonstandard-libs-oldinst.html" - - sh "./report.py --branches='${env.GITBRANCHES_FMI}' configs/conf.json" - sh "mv overview.html overview-fmi.html" - sh "./report.py --branches='${env.GITBRANCHES_FMI}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" - sh "mv overview.html overview-combined-fmi.html" - sh "./report.py --branches='${env.GITBRANCHES_FMI}' configs/conf-old.json" - sh "mv overview.html overview-old-libs-fmi.html" - sh "./report.py --branches='${env.GITBRANCHES_FMI}' configs/conf-nonstandard.json" - sh "mv overview.html overview-nonstandard-libs-fmi.html" - - sh "./report.py --branches='${env.GITBRANCHES_DAE}' configs/conf.json" - sh "mv overview.html overview-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_DAE}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" - sh "mv overview.html overview-combined-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_DAE}' configs/conf-old.json" - sh "mv overview.html overview-old-libs-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_DAE}' configs/conf-nonstandard.json" - sh "mv overview.html overview-nonstandard-libs-dae.html" - - sh "./report.py --branches='${env.GITBRANCHES_NEWBACKEND_DAE}' configs/conf.json" - sh "mv overview.html overview-newbackend-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWBACKEND_DAE}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" - sh "mv overview.html overview-combined-newbackend-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWBACKEND_DAE}' configs/conf-old.json" - sh "mv overview.html overview-old-libs-newbackend-dae.html" - sh "./report.py --branches='${env.GITBRANCHES_NEWBACKEND_DAE}' configs/conf-nonstandard.json" - sh "mv overview.html overview-nonstandard-libs-newbackend-dae.html" - - sh "./report.py --branches='cvode master' configs/conf.json" - sh "mv overview.html overview-cvode.html" - - sh "./report.py --branches='gbode master' configs/conf.json" - sh "mv overview.html overview-gbode.html" - - sh "./report.py --branches='ida master' configs/conf.json" - sh "mv overview.html overview-ida.html" - - // The three ways one wasm artifact is simulated, against master: they share - // an export, so only the simulation and the verification tell them apart. - sh "./report.py --branches='${env.GITBRANCHES_WASM_JIT} master' configs/conf.json" - sh "mv overview.html overview-wasm-jit.html" - - sh "./report.py --branches='${env.GITBRANCHES_CPP}' configs/conf.json" - sh "mv overview.html overview-c++.html" - - sh "./report.py --branches='${env.GITBRANCHES}' configs/conf.json" - - sh 'date' - sh 'find overview*.html history -type f | wc -l' - sh 'find overview*.html history' - - sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])]) + script { + report.prepare() + report.reportsAndPlots() + report.overviews() + report.publish() + } } } @@ -627,7 +552,7 @@ pipeline { } when { beforeAgent true - expression { pullRequest() } + expression { common.pullRequest() } } environment { PYTHONIOENCODING = 'utf-8' @@ -635,548 +560,10 @@ pipeline { } steps { script { - if (!(pullRequest() ==~ /[0-9]+/)) { - error "pull_request is a pull request number; got '${params.pull_request}'" - } - } - sh 'rm -rf history' - script { - def report = "./pr-report.py '${pullRequest()}' --baseline='${(params.pull_request_baseline ?: 'master').trim()}'" - if (params.pull_request_comment) { - // Whoever the token belongs to is who the comment comes from. - withCredentials([string(credentialsId: 'github-token', variable: 'GITHUB_TOKEN')]) { - sh "${report} --comment" - } - } else { - sh report - } + common.checkPullRequestNumber() + report.pullRequestReport(common.pullRequest(), params.pull_request_baseline, params.pull_request_comment ?: false) } - // The summary is in the build log as well, so a run without a token - // still leaves it somewhere to copy from. - sh 'cat history/pr-*/00_comment.md' - sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])]) } } } } -/** - * The pull request this job is testing, or "" when it is testing branches. - * - * A job only learns of a parameter that has been added to it once a build has - * run with the definition, so the first build after this file changes sees the - * new ones as null - which is every build of the pipeline, not only one asking - * for a pull request, because the stages that ignore them still have to decide - * whether to run. Everything that reads them therefore falls back to what the - * definition says the default is. - */ -def pullRequest() { - return (params.pull_request ?: '').trim() -} - -/** - * The cores of the node a build is running on, physical and logical, asked for - * the way the OpenModelica job asks (numPhysicalCPU in .CI/common.groovy - * there), so that a machine is built on as itself rather than as the machine - * the numbers were written for. - * - * Unlike that one the answer is not stashed in the environment: this pipeline - * is agent none and its stages run on several machines within one build, so a - * cached answer would be the first node's. An override set on the node itself - * is still honoured. - */ -def numPhysicalCPU() { - if (env.JENKINS_NUM_PHYSICAL_CPU) { - return env.JENKINS_NUM_PHYSICAL_CPU - } - return sh(script: 'lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l', returnStdout: true).trim() -} - -def numLogicalCPU() { - if (env.JENKINS_NUM_LOGICAL_CPU) { - return env.JENKINS_NUM_LOGICAL_CPU - } - return sh(script: 'nproc', returnStdout: true).trim() -} - -def omsimulatorHash() { - return 'master' -} -/** - * Installs the libraries a run tests, with the node's own omc, and returns the home directory - * holding them. `runSh` is how the conversion script is run: that one uses the omc that was just - * built, which is a binary of the image when the job has one. - */ -def installLibraries(boolean removePackageOrder, boolean conversionScript, name, String omhomeTestedOMC, Closure runSh) { - sh "rm -rf '${env.HOME}/saved_omc/libraries/.openmodelica/libraries'" - sh "mkdir -p '${env.HOME}/saved_omc/libraries/'" - sh "HOME='${env.HOME}/saved_omc/libraries/' /usr/bin/omc OpenModelicaLibraryTesting/.CI/installLibraries.mos" - if (removePackageOrder) { - sh "find '${env.HOME}/saved_omc/libraries/' -name package.order -exec rm '{}' ';'" - } - // These exist (better packaged? on the machines) - sh "rm -rf '${env.HOME}/saved_omc/libraries/ClaRa' '${env.HOME}/saved_omc/libraries/ClaRa_Obsolete' '${env.HOME}/saved_omc/libraries/TILMedia'" - sh "cp -ai /mnt/ReferenceFiles/ExtraLibs/packaged/* '${env.HOME}/saved_omc/libraries/'" - echo "installLibraries removePackageOrder: ${removePackageOrder} conversionScript: ${conversionScript} name: ${name}" - if (conversionScript) { - runSh(""" - cd '${WORKSPACE}/OpenModelicaLibraryTesting' - OPENMODELICAHOME="${omhomeTestedOMC}" ./conversionscript.py --diff --allowErrorsInDiff '${env.HOME}/saved_omc/libraries/.openmodelica/libraries' - scp converted-libraries/.openmodelica/libraries/*.diff 'libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}' - """) - return "${WORKSPACE}/OpenModelicaLibraryTesting/converted-libraries" - } else { - return "${env.HOME}/saved_omc/libraries" - } -} - - -/** - * `docker run` flags capping the container's cgroup at 90% of the node's RAM. test.py limits one - * test at a time, not the sum of the parallel ones. --memory-swap has to repeat the limit; docker - * reads 0 as "unset" and then allows swapping. - */ -def memoryLimitArgs() { - def mb = sh(script: '''awk '/^MemTotal:/ { print int($2 / 1024 * 0.85) }' /proc/meminfo''', - returnStdout: true).trim() - echo "Test container memory limit: ${mb} MB, no swap" - return "--memory=${mb}m --memory-swap=${mb}m" -} - -/** - * Runs `body` with the environment of the shared Rust compile cache of the OpenModelica job - * (OpenModelica/.CI/sccache/), which builds the same commits first. Hitting its keys needs the - * same rust settings (opt-level 2, -DRUST_OMC_THREADS=4 at the call site), the same toolchain - * (hence the same docker image) and SCCACHE_BASEDIRS set to the checkout directory, which is a - * subdirectory of the workspace here but the workspace itself there. - * - * The server is started by sccachePreamble() instead, since a build in a container cannot use - * one started outside it. - */ -def withSccache(Closure body) { - withCredentials([string(credentialsId: 'sccache-ci-secret-key', - variable: 'AWS_SECRET_ACCESS_KEY')]) { - withEnv(['RUSTC_WRAPPER=sccache', - 'SCCACHE_BUCKET=omc-sccache', - 'SCCACHE_ENDPOINT=https://sccache.openmodelica.org', - 'SCCACHE_REGION=auto', - 'SCCACHE_S3_USE_SSL=true', - 'AWS_ACCESS_KEY_ID=sccache-ci', - 'CARGO_INCREMENTAL=0', - 'CARGO_PROFILE_RELEASE_OPT_LEVEL=2', - "SCCACHE_BASEDIRS=${env.WORKSPACE}/OpenModelica"]) { - body() - } - } -} - -/** - * Prepended to a build running under withSccache: starts a server with the current environment - * (a running one keeps the one it was started with), or drops RUSTC_WRAPPER if there is no sccache. - */ -def sccachePreamble() { - return ''' - if command -v sccache > /dev/null; then - log="`mktemp`" - sccache --stop-server > /dev/null 2>&1 || true - SCCACHE_ERROR_LOG="$log" SCCACHE_LOG=warn sccache --start-server - sccache --show-stats - if grep -qiE "storage (write )?check failed|read-only storage|cache storage failed" "$log"; then - echo "WARNING: the sccache S3 backend is unusable; building without a shared cache:" >&2 - cat "$log" >&2 - fi - rm -f "$log" - else - echo "sccache was not found; building without a compile cache" - unset RUSTC_WRAPPER - fi - ''' -} - -/** - * Launches the test.py script with the given options. - * - * @param branch: OpenModelica branch to test. Will checkout the branch and build omc from it. - * @param name: Unique name of the library test. Passed to test.py via flag `--branch`. - * Also used for stashing omc and uploading results to https://test.openmodelica.org. - * @param extraFlags: Additional compiler flags passed to test.py via flag `--extraflags`. - * @param omsHash: OMSimulator SHA. - * @param omcompiler: Checkout old OMCompiler submodule. Should be `false` nowadays. - * @param extrasimflags: Additional simulation flags passed to test.py via flag `--extrasimflags`. - * @param testFlags: Additional flags passed to test.py verbatim, e.g. `--nobuildmodel`. - * @param removePackageOrder: Passed to `installLibraries`. - * @param conversionScript: Passed to `installLibraries`. - * @param jobs: The number of tests/jobs to launch in parallel. - * By default this is set to `0` which means launch as many tests as there are available - * physical cpus on the machine'. - * @param libs_config_file: The config file to be used for testing. - * This file specifies which libraries to test and what options to use for them. - * @param cmakeFlags: Target-specific cmake flags, e.g. `-DOM_OMC_ENABLE_RUST=ON`. If non-empty, omc is - * built with cmake instead of autotools; the shared release flags are added here. - * @param dockerfile: Directory with a Dockerfile, relative to the testing repository. Defaults to - * `.CI/testing`, the image every job runs in: the omc build, the OMSimulator - * build and test.py happen inside it, and only the steps using the node's own - * omc stay outside. Passing `''` runs the job on the node itself. - */ -/* The FMI simulators a job runs, from the parameters that used to start one job each. - * Both ticked is one job that builds every FMU once and simulates it with both, - * filling -fmi and -fmi-fmpy exactly as the two jobs did. */ -def fmiSimulators(boolean omsimulator, boolean fmpy) { - def simulators = [] - if (omsimulator) simulators << 'OMSimulator' - if (fmpy) simulators << 'fmpy' - return simulators -} - -def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null) { - sh ''' - find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true - - if test -z "$WORKSPACE"; then - echo "Odd workspace" - exit 1 - fi - ''' - - // Checked out before the omc build rather than just before test.py: it provides the Dockerfile. - sh """ - if test ! -d OpenModelicaLibraryTesting; then - git clone --recursive https://openmodelica.org/git-readonly/OpenModelicaLibraryTesting.git OpenModelicaLibraryTesting - fi - cd OpenModelicaLibraryTesting - git fetch - git reset --hard origin/master - """ - - def image = null - if (dockerfile) { - // Build context is the Dockerfile directory; the workspace has a directory per tested model. - sh "cp OpenModelicaLibraryTesting/requirements.txt OpenModelicaLibraryTesting/${dockerfile}/" - // Tagged after the Dockerfile rather than after the job, so that jobs sharing an image share - // its tag as well and a node keeps one of them instead of one per job it has ever run. - image = docker.build("openmodelica-library-testing:${dockerfile.tokenize('/').last()}", - "--pull OpenModelicaLibraryTesting/${dockerfile}") - } - // --init reaps the omc processes test.py orphans. ssh refuses to run for a uid it cannot look - // up, so the node's passwd entry is needed to publish the results. The home holds the cached omc - // build, the libraries (HOME during the test) and the ssh key; test.py writes a hash next to - // every reference file. rust-cargo-registry is the volume the OpenModelica job uses. - def dockerArgs = "--init" + - (image ? " ${memoryLimitArgs()}" : '') + - " -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro" + - " -v ${env.HOME}:${env.HOME}" + - " -v /mnt/ReferenceFiles:/mnt/ReferenceFiles" + - " --mount type=volume,source=rust-cargo-registry,target=/opt/rust/cargo/registry" - // Only in a container is the cgroup this job's own; a breach kills the greediest omc, which need - // not be the model that caused it. - def cgroupReport = image ? """ - cat /sys/fs/cgroup/memory.max || true - trap 'cat /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.events || true' EXIT - """ : '' - // Jenkins exports the node's environment into the container, hiding the image's. - def dockerEnv = ['PATH+VENV=/opt/libtest-venv/bin', - 'PATH+CARGO=/opt/rust/cargo/bin', - 'CARGO_HOME=/opt/rust/cargo', - 'RUSTUP_HOME=/opt/rust/rustup'] - // Runs a script in the image if the target uses one, on the node otherwise. - def runSh = { args -> - if (image) { - image.inside(dockerArgs) { withEnv(dockerEnv) { sh(args) } } - } else { - sh(args) - } - } - - // A job that does not say which simulators it wants gets the one its name implies. - def simulators = fmiSimulators - if (simulators == null) { - simulators = name.contains('fmpy') ? ['fmpy'] : (omsHash ? ['OMSimulator'] : []) - } - FMI_TESTING_FLAG = "" - if (simulators.contains('OMSimulator') && omsHash) { - // In the image rather than on the node: test.py runs this binary from inside it, and one built - // against the node's libraries need not load there. - runSh(""" - if ! test -d OMSimulator; then - git clone --recursive https://openmodelica.org/git-readonly/OMSimulator.git || exit 1 - fi - cd OMSimulator || exit 1 - - git fetch || exit 1 - git reset --hard "${omsHash}" || exit 1 - - git rev-parse HEAD > .newhash - echo "OMSimulator Hash: ${omsHash} and commit:" - cat .newhash || true - echo Old Hash: - cat ~/saved_omc/OMSimulator/.githash || true - - # The second test rebuilds a cached build that does not run here: one made on - # the node before the job moved into a container, or against an older image, - # matches by hash but misses the libraries it was linked against. - if ! cmp ~/saved_omc/OMSimulator/.githash .newhash || ! ~/saved_omc/OMSimulator/install/bin/OMSimulator --version; then - - git submodule sync --recursive || exit 1 - git clean -ffdx || exit 1 - git submodule update --init --recursive --force || exit 1 - git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 - cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=install/ - cmake --build build/ --target install || exit 1 - ./install/bin/OMSimulator --version || exit 1 - mkdir -p ~/saved_omc/OMSimulator || exit 1 - cp -a * ~/saved_omc/OMSimulator/ || exit 1 - git rev-parse HEAD > ~/saved_omc/OMSimulator/.githash || exit 1 - - fi - echo OMSimulator version: - ${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --version - """) - FMI_TESTING_FLAG = " --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator" - } - - if (simulators.contains('fmpy')) { - // The version is the image's; check it in the interpreter test.py will reach rather than - // installing one over it, which in a container would be thrown away with the container. - runSh('python3 -m fmpy -h > /dev/null || exit 1') - FMI_TESTING_FLAG += " --fmisimulator='python3 -m fmpy'" - } - - if (FMI_TESTING_FLAG) { - FMI_TESTING_FLAG = "--fmi=true${FMI_TESTING_FLAG} --default=ulimitExe=50" - if (name.contains('cvode')) { - FMI_TESTING_FLAG += " --fmuType=cs" - } - } - - // A pull request is fetched from GitHub itself: refs/pull/* is not on the - // read-only mirror the rest of the job clones. The merge ref rather than the - // head one, since the question is what happens once it is merged, and it is - // checked out detached so that nothing is left behind for the next run of the - // same workspace to trip over. - def pullRequest = branch.startsWith('pr/') && branch.substring(3).isInteger() ? branch.substring(3) : '' - def checkoutRef = pullRequest ? """ - REFS=`git ls-remote https://github.com/OpenModelica/OpenModelica.git "refs/pull/${pullRequest}/head" "refs/pull/${pullRequest}/merge"` || exit 1 - case "\$REFS" in - *"refs/pull/${pullRequest}/merge"*) - PRREF="refs/pull/${pullRequest}/merge" ;; - *"refs/pull/${pullRequest}/head"*) - # GitHub only has a merge ref while it can merge the pull request into - # its base branch. Without one there is still something to test, only it - # is the pull request on its own rather than as it would land. - echo "WARNING: pull request ${pullRequest} has no merge ref: it conflicts with its base branch, or it is closed." - echo "WARNING: testing refs/pull/${pullRequest}/head, which does not have what was merged into the base branch since it was branched." - PRREF="refs/pull/${pullRequest}/head" ;; - *) - echo "OpenModelica/OpenModelica has no pull request ${pullRequest}." - exit 1 ;; - esac - echo "Testing \$PRREF" - git fetch --force https://github.com/OpenModelica/OpenModelica.git "\$PRREF" || exit 1 - git checkout -f --detach FETCH_HEAD || exit 1 - git fetch --tags --force || exit 1 -""" : """ - git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 -""" - - OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" - - // The build runs in OMCompiler, one level below the cmake source tree. - if (cmakeFlags && omcompiler) { - error 'cmake builds need the OMCompiler directory of the OpenModelica repository (omcompiler=false)' - } - // The build used to say -j9, the cores of the machine this file was written - // for in 2019, and -j16, the cores of the ryzen-5950x machines that replaced - // it - the commit that raised the others to 16 left the omc build at 9. Named - // buildJobs rather than jobs: that is this method's own parameter, how many - // models test.py tests at a time. - def buildJobs = numPhysicalCPU() - echo "Building omc with -j${buildJobs} on ${env.NODE_NAME}" - - def buildOMC - if (cmakeFlags) { - buildOMC = sccachePreamble() + """ - cmake -S .. -B ../build_cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="`pwd`/build" \ - -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_Fortran_COMPILER=gfortran \ - -DCMAKE_C_FLAGS=-march=native -DCMAKE_CXX_FLAGS=-march=native \ - -DOM_USE_CCACHE=OFF -DOM_ENABLE_GUI_CLIENTS=OFF -DOM_ENABLE_OMSIMULATOR=OFF \ - ${cmakeFlags} || exit 1 - if ! time cmake --build ../build_cmake --parallel ${buildJobs} --target install > log 2>&1; then - cat log - exit 1 - fi - build/bin/omc --version || exit 1 - sccache --show-stats || true - """ - } else { - buildOMC = """ - autoreconf --install - ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omlibrary=all --with-omniORB - time make -j${buildJobs} clean - if ! time make -j${buildJobs} omc > log 2>&1; then - cat log - exit 1 - fi - if ! time make -j${buildJobs} runtimeCPPinstall > log 2>&1; then - cat log - if test "${name}" = "master"; then - exit 1 - else - echo "Ignoring failed C++ runtime" - fi - fi - """ - } - - sh ''' - FREE=`df -k --output=avail "$PWD" | tail -n1` # df -k not df -h - if test "$FREE" -lt 31457280; then # 30G = 30*1024*1024k - echo "Less than 30 GB free disk space" - exit 1 - fi; - ''' - - sh 'killall omc || true' - - def checkoutAndBuild = """ - if test ! -d OpenModelica; then - git clone --recursive https://openmodelica.org/git-readonly/OpenModelica.git OpenModelica - fi - if test ! -d OMCompiler; then - git clone --recursive https://openmodelica.org/git-readonly/OMCompiler.git OMCompiler - fi - if test ! -d OMLibraries; then - git clone --recursive https://openmodelica.org/git-readonly/OMLibraries.git OMLibraries - fi - cd OMLibraries - git fetch - git reset --hard origin/library-coverage - - cd ../OpenModelica - git fetch - rm -rf OMCompiler # Make sure the old submodule is not there - git reset --hard origin/master - git clean -fdx - - cd ${OMCPATH} - - if ! test -f ~/saved_omc/${name}/.nogit; then - ${checkoutRef} - git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 - git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 - git clean -fdxq || exit 1 - git submodule status --recursive - fi - - export OPENMODELICAHOME="`pwd`/build" - - git rev-parse --verify HEAD > .newhash - echo New Hash: - cat .newhash - echo Old Hash: - cat ~/saved_omc/${name}/.githash || true - REBUILD="" - if cmp ~/saved_omc/${name}/.githash .newhash || test -f ~/saved_omc/${name}/.nogit; then - rsync -a --delete ~/saved_omc/${name}/ build/ || exit 1 - echo "Restoring cached OMC version: ${name}, `cat ~/saved_omc/${name}/.githash`" - # The hash says the sources match, not that the binary runs here: a cache - # filled on the node before the jobs moved into a container, or before the - # image changed, holds an omc linked against libraries that are missing now. - # Without this check the run dies much later, when test.py asks the restored - # binary for its version and gets exit code 127. - if ! build/bin/omc --version; then - if test -f ~/saved_omc/${name}/.nogit; then - echo "The cached omc of ${name} does not run in this environment, and .nogit forbids rebuilding it." - exit 1 - fi - echo "The cached omc of ${name} does not run in this environment; rebuilding it." - REBUILD=1 - fi - else - REBUILD=1 - fi - if test -n "\$REBUILD"; then - ${buildOMC} - rm -rf ~/saved_omc/${name}/ - mkdir -p ~/saved_omc/${name}/ - CMD="rsync -a --delete build/ \$HOME/saved_omc/${name}/" - echo \$CMD - \$CMD || exit 1 - cp .newhash ~/saved_omc/${name}/.githash - fi - """ - - if (cmakeFlags) { - withSccache { runSh(checkoutAndBuild) } - } else { - runSh(checkoutAndBuild) - } - - sh """ - cd OpenModelica - rm -rf "`pwd`/${OMCPATH}/build/lib/omlibrary/" - mkdir -p "`pwd`/${OMCPATH}/build/lib/omlibrary/" - # (cd ../OMLibraries && git rev-parse HEAD) - # if ! time make -j16 -C ../OMLibraries all BUILD_DIR="`pwd`/${OMCPATH}/build/lib/omlibrary/" > log 2>&1; then - # cat log - # exit 1 - # fi - if ! time make -j${numLogicalCPU()} -C testsuite/ReferenceFiles > log 2>&1; then - cat log - exit 1 - fi - - cd ../ - rm -rf Reference-modelica.org - ln -s /mnt/ReferenceFiles/modelica.org Reference-modelica.org - """ - // sh 'rsync -av modelica-ro:/files/RegressionTesting/ReferenceResults Reference-modelica.org || true # exit 1' - - MSLREFERENCE="${WORKSPACE}/Reference-modelica.org/ReferenceResults" - REFERENCEFILES="${WORKSPACE}/OpenModelica/testsuite/ReferenceFiles" - GITREPOS="${WORKSPACE}/OpenModelica/libraries/git" - PNLIBREFS="/mnt/ReferenceFiles/PNlib/ReferenceFiles" - THERMOFLUIDSTREAMREFS="/mnt/ReferenceFiles/ThermofluidStream-main-regression/ReferenceData" - THERMOFLUIDSTREAMREFSOM="/mnt/ReferenceFiles/ThermofluidStream-OM-regression/ReferenceData" - - sh """ - test -f "${MSLREFERENCE}/MAP-LIB_ReferenceResults/v4.0.0/README.md" || exit 1 - - mkdir -p "/var/www/libraries.openmodelica.org/branches/${name}/" - """ - - def libraryPath = installLibraries(removePackageOrder, conversionScript, name, "${WORKSPACE}/OpenModelica/${OMCPATH}/build", runSh) - - sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'" - - sh 'date' - - // The password of the results database comes from a secret file, bound here - // rather than for the pipeline as a whole: writing the file needs the - // workspace of a node, and the pipeline has agent none. - withCredentials([file(credentialsId: 'omdb-pgpass', variable: 'PGPASSFILE')]) { - runSh(""" - export OPENMODELICAHOME="${WORKSPACE}/OpenModelica/${OMCPATH}/build" - export MSLREFERENCE="${MSLREFERENCE}" - export REFERENCEFILES="${REFERENCEFILES}" - export GITREPOS="${GITREPOS}" - export PNLIBREFS="${PNLIBREFS}" - export THERMOFLUIDSTREAMREFS="${THERMOFLUIDSTREAMREFS}" - export THERMOFLUIDSTREAMREFSOM="${THERMOFLUIDSTREAMREFSOM}" - export PREVIOUSHOME="${env.HOME}" - export HOME="${libraryPath}" - # we need to do some crap magic here to make sure python3 finds fmpy as we change the HOME here - # too bad if we cannot do it, just continue - ln -s -t \${HOME} \${PREVIOUSHOME}/.local .local || true - - ${cgroupReport} - cd OpenModelicaLibraryTesting - # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions - stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 - """) - sh 'date' - // In the image: the script talks to the results database through psycopg2, - // which is in the image's python environment and not on the node. - runSh("cd OpenModelicaLibraryTesting/ && ./clean-empty-omcversion-dates.py") - } - -} diff --git a/.CI/common.groovy b/.CI/common.groovy new file mode 100644 index 0000000..eb6de98 --- /dev/null +++ b/.CI/common.groovy @@ -0,0 +1,574 @@ +// The functions .CI/Jenkinsfile calls, kept out of it the way the OpenModelica +// repository keeps its own: the pipeline loads this file in its `setup` stage +// and reaches everything here as `common.()`. +// +// A file loaded this way shares the binding of the pipeline, so `params`, `env` +// and the steps (`sh`, `docker`, `withCredentials`, ...) are the same here as +// there. The load happens on one node and the object is used from every stage, +// which is what the OpenModelica pipeline does as well. + +/** + * The pull request this job is testing, or "" when it is testing branches. + * + * A job only learns of a parameter that has been added to it once a build has + * run with the definition, so the first build after this file changes sees the + * new ones as null - which is every build of the pipeline, not only one asking + * for a pull request, because the stages that ignore them still have to decide + * whether to run. Everything that reads them therefore falls back to what the + * definition says the default is. + */ +def pullRequest() { + return (params.pull_request ?: '').trim() +} + +/** + * Fails the build unless `pull_request` looks like a pull request number. + */ +def checkPullRequestNumber() { + if (!(pullRequest() ==~ /[0-9]+/)) { + error "pull_request is a pull request number; got '${params.pull_request}'" + } +} + +/** + * Fails the build unless OpenModelica/OpenModelica has the pull request + * `pull_request` names. Issues and pull requests share one numbering, so an + * issue number passes checkPullRequestNumber(), and only a pull request has + * refs/pull//* on GitHub. + * + * Asked before the clone, the reset and the build, which cost minutes before + * they arrive at the same answer. + */ +def checkPullRequestExists() { + sh """ + if ! git ls-remote --exit-code https://github.com/OpenModelica/OpenModelica.git 'refs/pull/${pullRequest()}/*' > /dev/null; then + echo "OpenModelica/OpenModelica has no pull request ${pullRequest()}. Issues and pull requests share one numbering there, so check that ${pullRequest()} is not the number of an issue." + exit 1 + fi + """ +} + +/** + * Removes the cached omc builds of the pull requests tested more than two weeks + * ago. One build per pull request is kept, as for a branch, and they accumulate: + * a pull request is tested once and never again. The second line is the layout + * of the runs before they moved under pr/. + */ +def removeStalePullRequestBuilds() { + sh ''' + find "$HOME/saved_omc/pr" -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} ";" 2> /dev/null || true + find "$HOME/saved_omc" -mindepth 1 -maxdepth 1 -name "pr-*" -type d -mtime +14 -exec rm -rf {} ";" || true + ''' +} + +/** + * The cores of the node a build is running on, physical and logical, asked for + * the way the OpenModelica job asks (numPhysicalCPU in .CI/common.groovy + * there), so that a machine is built on as itself rather than as the machine + * the numbers were written for. + * + * Unlike that one the answer is not stashed in the environment: this pipeline + * is agent none and its stages run on several machines within one build, so a + * cached answer would be the first node's. An override set on the node itself + * is still honoured. + */ +def numPhysicalCPU() { + if (env.JENKINS_NUM_PHYSICAL_CPU) { + return env.JENKINS_NUM_PHYSICAL_CPU + } + return sh(script: 'lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l', returnStdout: true).trim() +} + +def numLogicalCPU() { + if (env.JENKINS_NUM_LOGICAL_CPU) { + return env.JENKINS_NUM_LOGICAL_CPU + } + return sh(script: 'nproc', returnStdout: true).trim() +} + +/** + * Installs the libraries a run tests, with the node's own omc, and returns the home directory + * holding them. `runSh` is how the conversion script is run: that one uses the omc that was just + * built, which is a binary of the image when the job has one. + */ +def installLibraries(boolean removePackageOrder, boolean conversionScript, name, String omhomeTestedOMC, Closure runSh) { + sh "rm -rf '${env.HOME}/saved_omc/libraries/.openmodelica/libraries'" + sh "mkdir -p '${env.HOME}/saved_omc/libraries/'" + sh "HOME='${env.HOME}/saved_omc/libraries/' /usr/bin/omc OpenModelicaLibraryTesting/.CI/installLibraries.mos" + if (removePackageOrder) { + sh "find '${env.HOME}/saved_omc/libraries/' -name package.order -exec rm '{}' ';'" + } + // These exist (better packaged? on the machines) + sh "rm -rf '${env.HOME}/saved_omc/libraries/ClaRa' '${env.HOME}/saved_omc/libraries/ClaRa_Obsolete' '${env.HOME}/saved_omc/libraries/TILMedia'" + sh "cp -ai /mnt/ReferenceFiles/ExtraLibs/packaged/* '${env.HOME}/saved_omc/libraries/'" + echo "installLibraries removePackageOrder: ${removePackageOrder} conversionScript: ${conversionScript} name: ${name}" + if (conversionScript) { + runSh(""" + cd '${env.WORKSPACE}/OpenModelicaLibraryTesting' + OPENMODELICAHOME="${omhomeTestedOMC}" ./conversionscript.py --diff --allowErrorsInDiff '${env.HOME}/saved_omc/libraries/.openmodelica/libraries' + scp converted-libraries/.openmodelica/libraries/*.diff 'libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}' + """) + return "${env.WORKSPACE}/OpenModelicaLibraryTesting/converted-libraries" + } else { + return "${env.HOME}/saved_omc/libraries" + } +} + +/** + * `docker run` flags capping the container's cgroup at 90% of the node's RAM. test.py limits one + * test at a time, not the sum of the parallel ones. --memory-swap has to repeat the limit; docker + * reads 0 as "unset" and then allows swapping. + */ +def memoryLimitArgs() { + def mb = sh(script: '''awk '/^MemTotal:/ { print int($2 / 1024 * 0.85) }' /proc/meminfo''', + returnStdout: true).trim() + echo "Test container memory limit: ${mb} MB, no swap" + return "--memory=${mb}m --memory-swap=${mb}m" +} + +/** + * Runs `body` with the environment of the shared Rust compile cache of the OpenModelica job + * (OpenModelica/.CI/sccache/), which builds the same commits first. Hitting its keys needs the + * same rust settings (opt-level 2, -DRUST_OMC_THREADS=4 at the call site), the same toolchain + * (hence the same docker image) and SCCACHE_BASEDIRS set to the checkout directory, which is a + * subdirectory of the workspace here but the workspace itself there. + * + * The server is started by sccachePreamble() instead, since a build in a container cannot use + * one started outside it. + */ +def withSccache(Closure body) { + withCredentials([string(credentialsId: 'sccache-ci-secret-key', + variable: 'AWS_SECRET_ACCESS_KEY')]) { + withEnv(['RUSTC_WRAPPER=sccache', + 'SCCACHE_BUCKET=omc-sccache', + 'SCCACHE_ENDPOINT=https://sccache.openmodelica.org', + 'SCCACHE_REGION=auto', + 'SCCACHE_S3_USE_SSL=true', + 'AWS_ACCESS_KEY_ID=sccache-ci', + 'CARGO_INCREMENTAL=0', + 'CARGO_PROFILE_RELEASE_OPT_LEVEL=2', + "SCCACHE_BASEDIRS=${env.WORKSPACE}/OpenModelica"]) { + body() + } + } +} + +/** + * Prepended to a build running under withSccache: starts a server with the current environment + * (a running one keeps the one it was started with), or drops RUSTC_WRAPPER if there is no sccache. + */ +def sccachePreamble() { + return ''' + if command -v sccache > /dev/null; then + log="`mktemp`" + sccache --stop-server > /dev/null 2>&1 || true + SCCACHE_ERROR_LOG="$log" SCCACHE_LOG=warn sccache --start-server + sccache --show-stats + if grep -qiE "storage (write )?check failed|read-only storage|cache storage failed" "$log"; then + echo "WARNING: the sccache S3 backend is unusable; building without a shared cache:" >&2 + cat "$log" >&2 + fi + rm -f "$log" + else + echo "sccache was not found; building without a compile cache" + unset RUSTC_WRAPPER + fi + ''' +} + +/* The FMI simulators a job runs, from the parameters that used to start one job each. + * Both ticked is one job that builds every FMU once and simulates it with both, + * filling -fmi and -fmi-fmpy exactly as the two jobs did. */ +def fmiSimulators(boolean omsimulator, boolean fmpy) { + def simulators = [] + if (omsimulator) { + simulators << 'OMSimulator' + } + if (fmpy) { + simulators << 'fmpy' + } + return simulators +} + +/** + * Launches the test.py script with the given options. + * + * @param branch: OpenModelica branch to test. Will checkout the branch and build omc from it. + * @param name: Unique name of the library test. Passed to test.py via flag `--branch`. + * Also used for stashing omc and uploading results to https://test.openmodelica.org. + * @param extraFlags: Additional compiler flags passed to test.py via flag `--extraflags`. + * @param omsHash: The OMSimulator commit to build and simulate the FMUs with: a SHA, + * a tag, or a ref of the remote such as `origin/master`. Not a local + * branch name - `git fetch` does not update those, so resetting to one + * keeps whatever commit the workspace was left at. Empty tests no FMUs. + * @param omcompiler: Checkout old OMCompiler submodule. Should be `false` nowadays. + * @param extrasimflags: Additional simulation flags passed to test.py via flag `--extrasimflags`. + * @param testFlags: Additional flags passed to test.py verbatim, e.g. `--nobuildmodel`. + * @param removePackageOrder: Passed to `installLibraries`. + * @param conversionScript: Passed to `installLibraries`. + * @param jobs: The number of tests/jobs to launch in parallel. + * By default this is set to `0` which means launch as many tests as there are available + * physical cpus on the machine'. + * @param libsConfigFile: The config file to be used for testing. + * This file specifies which libraries to test and what options to use for them. + * @param cmakeFlags: Target-specific cmake flags, e.g. `-DOM_OMC_ENABLE_RUST=ON`. If non-empty, omc is + * built with cmake instead of autotools; the shared release flags are added here. + * @param dockerfile: Directory with a Dockerfile, relative to the testing repository. Defaults to + * `.CI/testing`, the image every job runs in: the omc build, the OMSimulator + * build and test.py happen inside it, and only the steps using the node's own + * omc stay outside. Passing `''` runs the job on the node itself. + */ +def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libsConfigFile = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null) { + sh ''' + find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true + + if test -z "$WORKSPACE"; then + echo "Odd workspace" + exit 1 + fi + ''' + + // Checked out before the omc build rather than just before test.py: it provides the Dockerfile. + sh """ + if test ! -d OpenModelicaLibraryTesting; then + git clone --recursive https://openmodelica.org/git-readonly/OpenModelicaLibraryTesting.git OpenModelicaLibraryTesting + fi + cd OpenModelicaLibraryTesting + git fetch + git reset --hard origin/master + """ + + def image = null + if (dockerfile) { + // Build context is the Dockerfile directory; the workspace has a directory per tested model. + sh "cp OpenModelicaLibraryTesting/requirements.txt OpenModelicaLibraryTesting/${dockerfile}/" + // Tagged after the Dockerfile rather than after the job, so that jobs sharing an image share + // its tag as well and a node keeps one of them instead of one per job it has ever run. + image = docker.build("openmodelica-library-testing:${dockerfile.tokenize('/').last()}", + "--pull OpenModelicaLibraryTesting/${dockerfile}") + } + // --init reaps the omc processes test.py orphans. ssh refuses to run for a uid it cannot look + // up, so the node's passwd entry is needed to publish the results. The home holds the cached omc + // build, the libraries (HOME during the test) and the ssh key; test.py writes a hash next to + // every reference file. rust-cargo-registry is the volume the OpenModelica job uses. + def dockerArgs = "--init" + + (image ? " ${memoryLimitArgs()}" : '') + + " -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro" + + " -v ${env.HOME}:${env.HOME}" + + " -v /mnt/ReferenceFiles:/mnt/ReferenceFiles" + + " --mount type=volume,source=rust-cargo-registry,target=/opt/rust/cargo/registry" + // Only in a container is the cgroup this job's own; a breach kills the greediest omc, which need + // not be the model that caused it. + def cgroupReport = image ? """ + cat /sys/fs/cgroup/memory.max || true + trap 'cat /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.events || true' EXIT + """ : '' + // Jenkins exports the node's environment into the container, hiding the image's. + def dockerEnv = ['PATH+VENV=/opt/libtest-venv/bin', + 'PATH+CARGO=/opt/rust/cargo/bin', + 'CARGO_HOME=/opt/rust/cargo', + 'RUSTUP_HOME=/opt/rust/rustup'] + // Runs a script in the image if the target uses one, on the node otherwise. + def runSh = { args -> + if (image) { + image.inside(dockerArgs) { withEnv(dockerEnv) { sh(args) } } + } else { + sh(args) + } + } + + // A job that does not say which simulators it wants gets the one its name implies. + def simulators = fmiSimulators + if (simulators == null) { + simulators = name.contains('fmpy') ? ['fmpy'] : (omsHash ? ['OMSimulator'] : []) + } + def FMI_TESTING_FLAG = "" + if (simulators.contains('OMSimulator') && omsHash) { + // In the image rather than on the node: test.py runs this binary from inside it, and one built + // against the node's libraries need not load there. + runSh(""" + if ! test -d OMSimulator; then + git clone --recursive https://openmodelica.org/git-readonly/OMSimulator.git || exit 1 + fi + cd OMSimulator || exit 1 + + git fetch || exit 1 + git reset --hard "${omsHash}" || exit 1 + + git rev-parse HEAD > .newhash + echo "OMSimulator Hash: ${omsHash} and commit:" + cat .newhash || true + echo Old Hash: + cat ~/saved_omc/OMSimulator/.githash || true + + # The second test rebuilds a cached build that does not run here: one made on + # the node before the job moved into a container, or against an older image, + # matches by hash but misses the libraries it was linked against. + if ! cmp ~/saved_omc/OMSimulator/.githash .newhash || ! ~/saved_omc/OMSimulator/install/bin/OMSimulator --version; then + + git submodule sync --recursive || exit 1 + git clean -ffdx || exit 1 + git submodule update --init --recursive --force || exit 1 + git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=install/ + cmake --build build/ --target install || exit 1 + ./install/bin/OMSimulator --version || exit 1 + mkdir -p ~/saved_omc/OMSimulator || exit 1 + cp -a * ~/saved_omc/OMSimulator/ || exit 1 + git rev-parse HEAD > ~/saved_omc/OMSimulator/.githash || exit 1 + + fi + echo OMSimulator version: + ${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --version + """) + FMI_TESTING_FLAG = " --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator" + } + + if (simulators.contains('fmpy')) { + // The version is the image's; check it in the interpreter test.py will reach rather than + // installing one over it, which in a container would be thrown away with the container. + runSh('python3 -m fmpy -h > /dev/null || exit 1') + FMI_TESTING_FLAG += " --fmisimulator='python3 -m fmpy'" + } + + if (FMI_TESTING_FLAG) { + FMI_TESTING_FLAG = "--fmi=true${FMI_TESTING_FLAG} --default=ulimitExe=50" + if (name.contains('cvode')) { + FMI_TESTING_FLAG += " --fmuType=cs" + } + } + + // A pull request is fetched from GitHub itself: refs/pull/* is not on the + // read-only mirror the rest of the job clones. The merge ref rather than the + // head one, since the question is what happens once it is merged, and it is + // checked out detached so that nothing is left behind for the next run of the + // same workspace to trip over. + def pullRequest = branch.startsWith('pr/') && branch.substring(3).isInteger() ? branch.substring(3) : '' + def checkoutRef = pullRequest ? """ + REFS=`git ls-remote https://github.com/OpenModelica/OpenModelica.git "refs/pull/${pullRequest}/head" "refs/pull/${pullRequest}/merge"` || exit 1 + case "\$REFS" in + *"refs/pull/${pullRequest}/merge"*) + PRREF="refs/pull/${pullRequest}/merge" ;; + *"refs/pull/${pullRequest}/head"*) + # GitHub only has a merge ref while it can merge the pull request into + # its base branch. Without one there is still something to test, only it + # is the pull request on its own rather than as it would land. + echo "WARNING: pull request ${pullRequest} has no merge ref: it conflicts with its base branch, or it is closed." + echo "WARNING: testing refs/pull/${pullRequest}/head, which does not have what was merged into the base branch since it was branched." + PRREF="refs/pull/${pullRequest}/head" ;; + *) + echo "OpenModelica/OpenModelica has no pull request ${pullRequest}." + exit 1 ;; + esac + echo "Testing \$PRREF" + git fetch --force https://github.com/OpenModelica/OpenModelica.git "\$PRREF" || exit 1 + git checkout -f --detach FETCH_HEAD || exit 1 + git fetch --tags --force || exit 1 +""" : """ + git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 +""" + + def OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" + + // The build runs in OMCompiler, one level below the cmake source tree. + if (cmakeFlags && omcompiler) { + error 'cmake builds need the OMCompiler directory of the OpenModelica repository (omcompiler=false)' + } + // The build used to say -j9, the cores of the machine this file was written + // for in 2019, and -j16, the cores of the ryzen-5950x machines that replaced + // it - the commit that raised the others to 16 left the omc build at 9. Named + // buildJobs rather than jobs: that is this method's own parameter, how many + // models test.py tests at a time. + def buildJobs = numPhysicalCPU() + echo "Building omc with -j${buildJobs} on ${env.NODE_NAME}" + + def buildOMC + if (cmakeFlags) { + buildOMC = sccachePreamble() + """ + cmake -S .. -B ../build_cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="`pwd`/build" \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_Fortran_COMPILER=gfortran \ + -DCMAKE_C_FLAGS=-march=native -DCMAKE_CXX_FLAGS=-march=native \ + -DOM_USE_CCACHE=OFF -DOM_ENABLE_GUI_CLIENTS=OFF -DOM_ENABLE_OMSIMULATOR=OFF \ + ${cmakeFlags} || exit 1 + if ! time cmake --build ../build_cmake --parallel ${buildJobs} --target install > log 2>&1; then + cat log + exit 1 + fi + build/bin/omc --version || exit 1 + sccache --show-stats || true + """ + } else { + buildOMC = """ + autoreconf --install + ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omlibrary=all --with-omniORB + time make -j${buildJobs} clean + if ! time make -j${buildJobs} omc > log 2>&1; then + cat log + exit 1 + fi + if ! time make -j${buildJobs} runtimeCPPinstall > log 2>&1; then + cat log + if test "${name}" = "master"; then + exit 1 + else + echo "Ignoring failed C++ runtime" + fi + fi + """ + } + + sh ''' + FREE=`df -k --output=avail "$PWD" | tail -n1` # df -k not df -h + if test "$FREE" -lt 31457280; then # 30G = 30*1024*1024k + echo "Less than 30 GB free disk space" + exit 1 + fi; + ''' + + sh 'killall omc || true' + + def checkoutAndBuild = """ + if test ! -d OpenModelica; then + git clone --recursive https://openmodelica.org/git-readonly/OpenModelica.git OpenModelica + fi + if test ! -d OMCompiler; then + git clone --recursive https://openmodelica.org/git-readonly/OMCompiler.git OMCompiler + fi + if test ! -d OMLibraries; then + git clone --recursive https://openmodelica.org/git-readonly/OMLibraries.git OMLibraries + fi + cd OMLibraries + git fetch + git reset --hard origin/library-coverage + + cd ../OpenModelica + git fetch + rm -rf OMCompiler # Make sure the old submodule is not there + git reset --hard origin/master + git clean -fdx + + cd ${OMCPATH} + + if ! test -f ~/saved_omc/${name}/.nogit; then + ${checkoutRef} + git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 + git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 + git clean -fdxq || exit 1 + git submodule status --recursive + fi + + export OPENMODELICAHOME="`pwd`/build" + + git rev-parse --verify HEAD > .newhash + echo New Hash: + cat .newhash + echo Old Hash: + cat ~/saved_omc/${name}/.githash || true + REBUILD="" + if cmp ~/saved_omc/${name}/.githash .newhash || test -f ~/saved_omc/${name}/.nogit; then + rsync -a --delete ~/saved_omc/${name}/ build/ || exit 1 + echo "Restoring cached OMC version: ${name}, `cat ~/saved_omc/${name}/.githash`" + # The hash says the sources match, not that the binary runs here: a cache + # filled on the node before the jobs moved into a container, or before the + # image changed, holds an omc linked against libraries that are missing now. + # Without this check the run dies much later, when test.py asks the restored + # binary for its version and gets exit code 127. + if ! build/bin/omc --version; then + if test -f ~/saved_omc/${name}/.nogit; then + echo "The cached omc of ${name} does not run in this environment, and .nogit forbids rebuilding it." + exit 1 + fi + echo "The cached omc of ${name} does not run in this environment; rebuilding it." + REBUILD=1 + fi + else + REBUILD=1 + fi + if test -n "\$REBUILD"; then + ${buildOMC} + rm -rf ~/saved_omc/${name}/ + mkdir -p ~/saved_omc/${name}/ + CMD="rsync -a --delete build/ \$HOME/saved_omc/${name}/" + echo \$CMD + \$CMD || exit 1 + cp .newhash ~/saved_omc/${name}/.githash + fi + """ + + if (cmakeFlags) { + withSccache { runSh(checkoutAndBuild) } + } else { + runSh(checkoutAndBuild) + } + + sh """ + cd OpenModelica + rm -rf "`pwd`/${OMCPATH}/build/lib/omlibrary/" + mkdir -p "`pwd`/${OMCPATH}/build/lib/omlibrary/" + # (cd ../OMLibraries && git rev-parse HEAD) + # if ! time make -j16 -C ../OMLibraries all BUILD_DIR="`pwd`/${OMCPATH}/build/lib/omlibrary/" > log 2>&1; then + # cat log + # exit 1 + # fi + if ! time make -j${numLogicalCPU()} -C testsuite/ReferenceFiles > log 2>&1; then + cat log + exit 1 + fi + + cd ../ + rm -rf Reference-modelica.org + ln -s /mnt/ReferenceFiles/modelica.org Reference-modelica.org + """ + // sh 'rsync -av modelica-ro:/files/RegressionTesting/ReferenceResults Reference-modelica.org || true # exit 1' + + def MSLREFERENCE = "${env.WORKSPACE}/Reference-modelica.org/ReferenceResults" + def REFERENCEFILES = "${env.WORKSPACE}/OpenModelica/testsuite/ReferenceFiles" + def GITREPOS = "${env.WORKSPACE}/OpenModelica/libraries/git" + def PNLIBREFS = "/mnt/ReferenceFiles/PNlib/ReferenceFiles" + def THERMOFLUIDSTREAMREFS = "/mnt/ReferenceFiles/ThermofluidStream-main-regression/ReferenceData" + def THERMOFLUIDSTREAMREFSOM = "/mnt/ReferenceFiles/ThermofluidStream-OM-regression/ReferenceData" + + sh """ + test -f "${MSLREFERENCE}/MAP-LIB_ReferenceResults/v4.0.0/README.md" || exit 1 + + mkdir -p "/var/www/libraries.openmodelica.org/branches/${name}/" + """ + + def libraryPath = installLibraries(removePackageOrder, conversionScript, name, "${env.WORKSPACE}/OpenModelica/${OMCPATH}/build", runSh) + + sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'" + + sh 'date' + + // The password of the results database comes from a secret file, bound here + // rather than for the pipeline as a whole: writing the file needs the + // workspace of a node, and the pipeline has agent none. + withCredentials([file(credentialsId: 'omdb-pgpass', variable: 'PGPASSFILE')]) { + runSh(""" + export OPENMODELICAHOME="${env.WORKSPACE}/OpenModelica/${OMCPATH}/build" + export MSLREFERENCE="${MSLREFERENCE}" + export REFERENCEFILES="${REFERENCEFILES}" + export GITREPOS="${GITREPOS}" + export PNLIBREFS="${PNLIBREFS}" + export THERMOFLUIDSTREAMREFS="${THERMOFLUIDSTREAMREFS}" + export THERMOFLUIDSTREAMREFSOM="${THERMOFLUIDSTREAMREFSOM}" + export PREVIOUSHOME="${env.HOME}" + export HOME="${libraryPath}" + # we need to do some crap magic here to make sure python3 finds fmpy as we change the HOME here + # too bad if we cannot do it, just continue + ln -s -t \${HOME} \${PREVIOUSHOME}/.local .local || true + + ${cgroupReport} + cd OpenModelicaLibraryTesting + # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions + stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libsConfigFile} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 + """) + sh 'date' + // In the image: the script talks to the results database through psycopg2, + // which is in the image's python environment and not on the node. + runSh("cd OpenModelicaLibraryTesting/ && ./clean-empty-omcversion-dates.py") + } +} + +return this diff --git a/.CI/report.groovy b/.CI/report.groovy new file mode 100644 index 0000000..bf67d40 --- /dev/null +++ b/.CI/report.groovy @@ -0,0 +1,157 @@ +// The report stage of .CI/Jenkinsfile, which turns the results of the runs into +// the pages published on libraries.openmodelica.org. Loaded next to +// .CI/common.groovy and reached as `report.()`. +// +// The branches a page covers are the GITBRANCHES* of the stage's environment; +// the functions here read them from `env` rather than taking them as arguments, +// so that the lists stay where the pipeline declares them. + +/** + * The three configuration files together: the standard libraries, the outdated + * ones and the nonstandard ones. Passing them to one report.py run is what + * makes a "combined" page. + */ +def allConfigs() { + return 'configs/conf.json configs/conf-old.json configs/conf-nonstandard.json' +} + +/** + * Every branch a run of all-reports.py and all-plots.py covers. + */ +def allBranches() { + return [env.GITBRANCHES, + env.GITBRANCHES_FMI, + env.GITBRANCHES_NEWINST, + env.GITBRANCHES_DAE, + env.GITBRANCHES_NEWBACKEND_DAE, + env.GITBRANCHES_CPP, + env.GITBRANCHES_WASM_JIT, + // The jobs that are a table of their own rather than one of the lists above. + 'conversion', + 'heavy_tests', + 'generateSymbolicJacobian', + 'gbode', + 'cvode', + 'ida'].join(' ') +} + +/** + * The workspace the pages are built in: the ones of the previous run are + * removed, the OpenModelica clone that all-reports.py reads the commits from is + * updated, and the omcversion rows of runs that produced no results are dropped. + */ +def prepare() { + sh 'rm -rf *.html history' + sh ''' + if ! test -d OpenModelica; then + git clone https://openmodelica.org/git-readonly/OpenModelica.git + fi + cd OpenModelica + git fetch + ''' + sh './clean-empty-omcversion-dates.py' +} + +/** + * The per-library reports and the plots of every branch. + */ +def reportsAndPlots() { + sh "./all-reports.py --email --omcgitdir=OpenModelica ${allBranches()}" + sh "./all-plots.py ${allBranches()}" +} + +/** + * One overview page. report.py always writes overview.html, so a page is that + * file under the name the publisher uploads it as. + * + * @param name: File name of the page, e.g. `overview-fmi.html`. + * @param branches: The branches it holds a column for. + * @param configs: The configuration files it covers, the standard libraries + * by default. + */ +def overview(String name, String branches, String configs = 'configs/conf.json') { + sh "./report.py --branches='${branches}' ${configs}" + sh "mv overview.html ${name}" +} + +/** + * The four pages a set of branches gets: the standard libraries, all three + * configurations combined, the outdated libraries and the nonstandard ones. + * `suffix` is what tells the four sets apart, e.g. `-fmi`. + */ +def overviewSet(String suffix, String branches) { + overview("overview${suffix}.html", branches) + overview("overview-combined${suffix}.html", branches, allConfigs()) + overview("overview-old-libs${suffix}.html", branches, 'configs/conf-old.json') + overview("overview-nonstandard-libs${suffix}.html", branches, 'configs/conf-nonstandard.json') +} + +/** + * Every overview page of the report stage. + */ +def overviews() { + def standard = "${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}" + overview('overview-combined.html', standard, allConfigs()) + overview('overview-old-libs.html', standard, 'configs/conf-old.json') + overview('overview-nonstandard-libs.html', standard, 'configs/conf-nonstandard.json') + overview('overview-special-jobs.html', "${env.GITBRANCHES_SPECIAL} conversion") + overview('overview-generateSymbolicJacobian.html', 'generateSymbolicJacobian') + overview('overview-heavy_tests.html', 'heavy_tests', 'configs/heavy_tests.json') + overview('overview-cvode.html', 'cvode master') + overview('overview-gbode.html', 'gbode master') + overview('overview-ida.html', 'ida master') + // The three ways one wasm artifact is simulated, against master: they share + // an export, so only the simulation and the verification tell them apart. + overview('overview-wasm-jit.html', "${env.GITBRANCHES_WASM_JIT} master") + overview('overview-c++.html', env.GITBRANCHES_CPP) + + overviewSet('-oldinst', env.GITBRANCHES_NEWINST) + overviewSet('-fmi', env.GITBRANCHES_FMI) + overviewSet('-dae', env.GITBRANCHES_DAE) + overviewSet('-newbackend-dae', env.GITBRANCHES_NEWBACKEND_DAE) + + // Last, and the one page that keeps the name report.py gives it: overview.html + // is what the site links to. + sh "./report.py --branches='${env.GITBRANCHES}' configs/conf.json" +} + +/** + * Uploads the pages, after saying how many of them there are and which. + */ +def publish() { + sh 'date' + sh 'find overview*.html history -type f | wc -l' + sh 'find overview*.html history' + + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])]) +} + +/** + * The report of one pull request run, which the report stage above does not + * cover: that one compares a branch against its own previous run, which a pull + * request has none of. + * + * @param pullRequest: The number of the tested pull request. + * @param baseline: The branch its results are compared against. + * @param comment: Post the summary as a comment on the pull request. Needs + * the github-token credential; whoever the token belongs to + * is who the comment comes from. + */ +def pullRequestReport(String pullRequest, String baseline, boolean comment) { + sh 'rm -rf history' + def prReport = "./pr-report.py '${pullRequest}' --baseline='${(baseline ?: 'master').trim()}'" + if (comment) { + withCredentials([string(credentialsId: 'github-token', variable: 'GITHUB_TOKEN')]) { + sh "${prReport} --comment" + } + } else { + sh prReport + } + // The summary is in the build log as well, so a run without a token still + // leaves it somewhere to copy from. + sh 'cat history/pr-*/00_comment.md' + + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])]) +} + +return this diff --git a/.github/workflows/lint-groovy.yml b/.github/workflows/lint-groovy.yml new file mode 100644 index 0000000..5dc4d1a --- /dev/null +++ b/.github/workflows/lint-groovy.yml @@ -0,0 +1,37 @@ +name: Lint Groovy + +on: + pull_request: + push: + branches: + - master + +jobs: + npm-groovy-lint: + name: Lint .CI/Jenkinsfile and .CI/common.groovy + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Java + uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 + with: + distribution: temurin + java-version: '21' + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - name: Install npm-groovy-lint + run: npm install --global npm-groovy-lint@18 + + # Lints every Jenkinsfile and *.groovy of the repository (the default file + # pattern), with the rules in .groovylintrc.json. A syntax error, which is + # otherwise found by a Jenkins build that is already running, is reported + # here as an error; warnings fail the job as well, and the info-level + # findings that remain are style this repository does not follow. + - name: Run npm-groovy-lint + run: npm-groovy-lint --loglevel warning --failon warning --no-insight diff --git a/.gitignore b/.gitignore index 388d96d..3040203 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__ /*.files /*.html /*.json +!/.groovylintrc.json /converted-libraries /MyLibrary*/ /ReferenceFiles/ diff --git a/.groovylintrc.json b/.groovylintrc.json new file mode 100644 index 0000000..a25f8da --- /dev/null +++ b/.groovylintrc.json @@ -0,0 +1,32 @@ +{ + "extends": "recommended-jenkinsfile", + "rules": { + "DuplicateNumberLiteral": { + "enabled": false + }, + "DuplicateStringLiteral": { + "enabled": false + }, + "Indentation": { + "spacesPerIndentLevel": 2 + }, + "LineLength": { + "enabled": false + }, + "MethodParameterTypeRequired": { + "enabled": false + }, + "MethodSize": { + "enabled": false + }, + "ParameterCount": { + "maxParameters": 15 + }, + "UnnecessaryElseStatement": { + "enabled": false + }, + "UnnecessaryGString": { + "enabled": false + } + } +} diff --git a/README.md b/README.md index fd5284b..203ed17 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,16 @@ OpenModelica [issue tracker](https://github.com/OpenModelica/OpenModelica/issues/new/choose) and ask us to do it for you. +### The pipeline + +[.CI/Jenkinsfile](.CI/Jenkinsfile) holds the stages; what they do is a function in +[.CI/common.groovy](.CI/common.groovy), or in [.CI/report.groovy](.CI/report.groovy) +for the pages published from the results. The `setup` stage loads both, and the +others reach them as `common.()` and `report.()`. All of it is +linted on every pull request by +[.github/workflows/lint-groovy.yml](.github/workflows/lint-groovy.yml), so a +Jenkinsfile that does not parse is found before a build runs it. + ### The image the OSMC jobs run in Every job of [.CI/Jenkinsfile](.CI/Jenkinsfile) runs in one docker image, built From 968baf2dbe066ae51e335d813b995595ff388f99 Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:09:09 +0200 Subject: [PATCH 2/4] Pin the GitHub actions by SHA An action named by a tag is whatever that tag points at on the day the workflow runs. The workflows name the commit now, with the release it belongs to in a comment, and every action is updated to its newest release while at it: checkout v4 to v7.0.1, setup-python v5 to v7.0.0, upload-artifact v4 to v7.0.1 and setup-openmodelica v1 to v1.1.0. Co-Authored-By: Claude Opus 5 --- .github/workflows/check_json.yml | 4 ++-- .github/workflows/test.yml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check_json.yml b/.github/workflows/check_json.yml index e9f0463..b617ef1 100644 --- a/.github/workflows/check_json.yml +++ b/.github/workflows/check_json.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python3 - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - name: Validate JSON files run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 852e4b9..9b9b0a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,10 +25,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup OpenModelica - uses: OpenModelica/setup-openmodelica@v1 + uses: OpenModelica/setup-openmodelica@dd6dc5fdb91c936dd3ab328902808c8a5421f146 # v1.1.0 with: version: ${{ matrix.omc-version }} packages: | @@ -45,7 +45,7 @@ jobs: fi - name: Setup Python3 - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' # caching pip dependencies @@ -81,7 +81,7 @@ jobs: fi - name: Archive sqlite3.db - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: ${{ matrix.os }}-${{ matrix.omc-version }}-sqlite3.db @@ -89,7 +89,7 @@ jobs: sqlite3.db - name: Archive HTML - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: ${{ matrix.os }}-${{ matrix.omc-version }}-MyLibrary.html From ab93c928a12a984e8d556275574066d585e2d4b6 Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:20:33 +0200 Subject: [PATCH 3/4] Remove the build machinery nothing uses any more runRegressiontest took an `omcompiler` flag choosing between the compiler directory of the OpenModelica repository and a standalone OMCompiler clone next to it, the layout from before the compiler moved into the repository. Every one of the 24 call sites passed false, so the parameter, the OMCPATH it picked, the guard against combining it with a cmake build, the clone of OMCompiler.git and the `rm -rf OMCompiler` that removed a leftover submodule directory are gone. OMLibraries was cloned with all its submodules, fetched and reset on every run for a `make -C ../OMLibraries` that has been commented out for as long: the libraries a run tests are the ones installLibraries() installs through the OpenModelica package manager. The clone is gone with it. The same goes for the omlibrary leftovers: `--with-omlibrary=all` is not an option of configure any more - configure.ac has one AC_ARG_WITH, cppruntime - and build/lib/omlibrary is not where a build puts libraries any more, so emptying it removed nothing and created an empty directory nothing reads. Co-Authored-By: Claude Opus 5 --- .CI/Jenkinsfile | 48 +++++++++++++++++++++++------------------------ .CI/common.groovy | 38 +++++++------------------------------ 2 files changed, 31 insertions(+), 55 deletions(-) diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index ced4bb1..5a39831 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -87,7 +87,7 @@ pipeline { expression { params.v1_26 } } steps { - script { common.runRegressiontest('maintenance/v1.26', 'v1.26', '', '', false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.26', 'v1.26', '', '', '', '', false, false) } } } @@ -104,7 +104,7 @@ pipeline { expression { params.v1_27 } } steps { - script { common.runRegressiontest('maintenance/v1.27', 'v1.27', '', '', false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.27', 'v1.27', '', '', '', '', false, false) } } } @@ -121,7 +121,7 @@ pipeline { expression { params.master } } steps { - script { common.runRegressiontest('master', 'master', '', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'master', '', '', '', '', false, false) } } } @@ -138,7 +138,7 @@ pipeline { expression { params.conversion_script } } steps { - script { common.runRegressiontest('master', 'conversion', '', '', false, '', '', false, true) } + script { common.runRegressiontest('master', 'conversion', '', '', '', '', false, true) } } } @@ -159,7 +159,7 @@ pipeline { common.checkPullRequestNumber() common.checkPullRequestExists() common.removeStalePullRequestBuilds() - common.runRegressiontest("pr/${common.pullRequest()}", "pr/${common.pullRequest()}", '', '', false, '', '', false, false, 0, params.pull_request_config ?: 'configs/conf.json') + common.runRegressiontest("pr/${common.pullRequest()}", "pr/${common.pullRequest()}", '', '', '', '', false, false, 0, params.pull_request_config ?: 'configs/conf.json') } } } @@ -177,7 +177,7 @@ pipeline { expression { params.newInst_newBackend } } steps { - script { common.runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'newInst-newBackend', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend")', '', '', '', false, false) } } } @@ -194,7 +194,7 @@ pipeline { expression { params.fmi_v1_26 || params.fmpy_fmi_v1_26 } } steps { - script { common.runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) } + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorRef, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) } } } stage('v1.27 FMI') { @@ -210,7 +210,7 @@ pipeline { expression { params.fmi_v1_27 || params.fmpy_fmi_v1_27 } } steps { - script { common.runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) } + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorRef, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) } } } stage('master FMI') { @@ -226,7 +226,7 @@ pipeline { expression { params.fmi_master || params.fmpy_fmi_master } } steps { - script { common.runRegressiontest('master', 'master-fmi', '', omsimulatorRef, false, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) } + script { common.runRegressiontest('master', 'master-fmi', '', omsimulatorRef, '', '', false, false, 0, 'configs/conf.json', '', '', common.fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) } } } @@ -243,7 +243,7 @@ pipeline { expression { params.cs_fmu_cvode_v1_26 } } steps { - script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, '', '', false, false) } } } stage('v1.27 CVODE CS-FMUs with OMSimulator') { @@ -259,7 +259,7 @@ pipeline { expression { params.cs_fmu_cvode_v1_27 } } steps { - script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, '', '', false, false) } } } stage('master CVODE CS-FMUs with OMSimulator') { @@ -275,7 +275,7 @@ pipeline { expression { params.cs_fmu_cvode_master } } steps { - script { common.runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, false, '', '', false, false) } + script { common.runRegressiontest('master', 'master-cs-fmu-cvode', 'setCommandLineOptions("--fmiFlags=s:cvode --fmuRuntimeDepends=modelica")', omsimulatorRef, '', '', false, false) } } } @@ -292,7 +292,7 @@ pipeline { expression { params.newInst_daeMode } } steps { - script { common.runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'newInst-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --daeMode=true")', '', '', '', false, false) } } } stage('newBackend-daeMode') { @@ -308,7 +308,7 @@ pipeline { expression { params.newBackend_daeMode } } steps { - script { common.runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'newBackend-daeMode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck --newBackend --daeMode=true")', '', '', '', false, false) } } } stage('oldInst') { @@ -324,7 +324,7 @@ pipeline { expression { params.oldInst } } steps { - script { common.runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'oldInst', 'setCommandLineOptions("-d=nonewInst")', '', '', '', false, false) } } } stage('cvode') { @@ -340,7 +340,7 @@ pipeline { expression { params.cvode } } steps { - script { common.runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s cvode', '', false, false) } + script { common.runRegressiontest('master', 'cvode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', '-s cvode', '', false, false) } } } stage('gbode') { @@ -356,7 +356,7 @@ pipeline { expression { params.gbode } } steps { - script { common.runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s gbode -gbm=radauIIA3', '', false, false) } + script { common.runRegressiontest('master', 'gbode', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', '-s gbode -gbm=radauIIA3', '', false, false) } } } stage('ida') { @@ -372,7 +372,7 @@ pipeline { expression { params.ida } } steps { - script { common.runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', false, '-s ida', '', false, false) } + script { common.runRegressiontest('master', 'ida', 'setCommandLineOptions("-d=newInst,-frontEndUnitCheck")', '', '-s ida', '', false, false) } } } stage('wasm-jit') { @@ -389,7 +389,7 @@ pipeline { } steps { script { - common.runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, '', '--wasmjitrunner=sim,me,cs', false, false, 0, 'configs/conf.json', + common.runRegressiontest('master', 'wasm-jit', 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', '', '--wasmjitrunner=sim,me,cs', false, false, 0, 'configs/conf.json', '-DOM_OMC_ENABLE_RUST=ON -DRUST_OMC_CI=ON -DRUST_OMC_THREADS=4 -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache') } } @@ -407,7 +407,7 @@ pipeline { expression { params.generateSymbolicJacobian } } steps { - script { common.runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'generateSymbolicJacobian', 'setCommandLineOptions("--generateSymbolicJacobian")', '', '', '', false, false) } } } stage('heavy_tests') { @@ -423,7 +423,7 @@ pipeline { expression { params.heavy_tests } } steps { - script { common.runRegressiontest('master', 'heavy_tests', '', '', false, '', '', false, false, 1, 'configs/heavy_tests.json') } + script { common.runRegressiontest('master', 'heavy_tests', '', '', '', '', false, false, 1, 'configs/heavy_tests.json') } } } @@ -440,7 +440,7 @@ pipeline { expression { params.cpp_v1_26 } } steps { - script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.26', 'v1.26-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', '', '', false, false) } } } stage('C++ v1.27') { @@ -456,7 +456,7 @@ pipeline { expression { params.cpp_v1_27 } } steps { - script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } + script { common.runRegressiontest('maintenance/v1.27', 'v1.27-cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', '', '', false, false) } } } @@ -473,7 +473,7 @@ pipeline { expression { params.cpp } } steps { - script { common.runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', false, '', '', false, false) } + script { common.runRegressiontest('master', 'cpp', 'setCommandLineOptions("--simCodeTarget=Cpp")', '', '', '', false, false) } } } } diff --git a/.CI/common.groovy b/.CI/common.groovy index eb6de98..6a0fa42 100644 --- a/.CI/common.groovy +++ b/.CI/common.groovy @@ -201,7 +201,6 @@ def fmiSimulators(boolean omsimulator, boolean fmpy) { * a tag, or a ref of the remote such as `origin/master`. Not a local * branch name - `git fetch` does not update those, so resetting to one * keeps whatever commit the workspace was left at. Empty tests no FMUs. - * @param omcompiler: Checkout old OMCompiler submodule. Should be `false` nowadays. * @param extrasimflags: Additional simulation flags passed to test.py via flag `--extrasimflags`. * @param testFlags: Additional flags passed to test.py verbatim, e.g. `--nobuildmodel`. * @param removePackageOrder: Passed to `installLibraries`. @@ -218,7 +217,7 @@ def fmiSimulators(boolean omsimulator, boolean fmpy) { * build and test.py happen inside it, and only the steps using the node's own * omc stay outside. Passing `''` runs the job on the node itself. */ -def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libsConfigFile = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null) { +def runRegressiontest(branch, name, extraFlags, omsHash, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libsConfigFile = 'configs/conf.json', cmakeFlags = '', dockerfile = '.CI/testing', fmiSimulators = null) { sh ''' find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true @@ -368,12 +367,6 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 """ - def OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" - - // The build runs in OMCompiler, one level below the cmake source tree. - if (cmakeFlags && omcompiler) { - error 'cmake builds need the OMCompiler directory of the OpenModelica repository (omcompiler=false)' - } // The build used to say -j9, the cores of the machine this file was written // for in 2019, and -j16, the cores of the ryzen-5950x machines that replaced // it - the commit that raised the others to 16 left the omc build at 9. Named @@ -384,6 +377,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla def buildOMC if (cmakeFlags) { + // Run from OpenModelica/OMCompiler, one directory below the cmake source tree. buildOMC = sccachePreamble() + """ cmake -S .. -B ../build_cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="`pwd`/build" \ @@ -401,7 +395,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla } else { buildOMC = """ autoreconf --install - ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omlibrary=all --with-omniORB + ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omniORB time make -j${buildJobs} clean if ! time make -j${buildJobs} omc > log 2>&1; then cat log @@ -432,23 +426,12 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla if test ! -d OpenModelica; then git clone --recursive https://openmodelica.org/git-readonly/OpenModelica.git OpenModelica fi - if test ! -d OMCompiler; then - git clone --recursive https://openmodelica.org/git-readonly/OMCompiler.git OMCompiler - fi - if test ! -d OMLibraries; then - git clone --recursive https://openmodelica.org/git-readonly/OMLibraries.git OMLibraries - fi - cd OMLibraries - git fetch - git reset --hard origin/library-coverage - - cd ../OpenModelica + cd OpenModelica git fetch - rm -rf OMCompiler # Make sure the old submodule is not there git reset --hard origin/master git clean -fdx - cd ${OMCPATH} + cd OMCompiler if ! test -f ~/saved_omc/${name}/.nogit; then ${checkoutRef} @@ -504,13 +487,6 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla sh """ cd OpenModelica - rm -rf "`pwd`/${OMCPATH}/build/lib/omlibrary/" - mkdir -p "`pwd`/${OMCPATH}/build/lib/omlibrary/" - # (cd ../OMLibraries && git rev-parse HEAD) - # if ! time make -j16 -C ../OMLibraries all BUILD_DIR="`pwd`/${OMCPATH}/build/lib/omlibrary/" > log 2>&1; then - # cat log - # exit 1 - # fi if ! time make -j${numLogicalCPU()} -C testsuite/ReferenceFiles > log 2>&1; then cat log exit 1 @@ -535,7 +511,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla mkdir -p "/var/www/libraries.openmodelica.org/branches/${name}/" """ - def libraryPath = installLibraries(removePackageOrder, conversionScript, name, "${env.WORKSPACE}/OpenModelica/${OMCPATH}/build", runSh) + def libraryPath = installLibraries(removePackageOrder, conversionScript, name, "${env.WORKSPACE}/OpenModelica/OMCompiler/build", runSh) sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'" @@ -546,7 +522,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla // workspace of a node, and the pipeline has agent none. withCredentials([file(credentialsId: 'omdb-pgpass', variable: 'PGPASSFILE')]) { runSh(""" - export OPENMODELICAHOME="${env.WORKSPACE}/OpenModelica/${OMCPATH}/build" + export OPENMODELICAHOME="${env.WORKSPACE}/OpenModelica/OMCompiler/build" export MSLREFERENCE="${MSLREFERENCE}" export REFERENCEFILES="${REFERENCEFILES}" export GITREPOS="${GITREPOS}" From b59976c76a1f0adea2b33b316c273d32292035bd Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:38:52 +0200 Subject: [PATCH 4/4] Take the omc build of runRegressiontest apart into functions One shell string of sixty lines checked out the sources, decided whether the cached build was still good and built omc when it was not, with the build itself interpolated into it from a second string a branch above. It is four functions now: - checkoutOMC() clones, resets and checks out the branch or the pull request ref, and updates the submodules. - buildOrRestoreOMC() decides between the cache in ~/saved_omc/ and a build, and runs the one it picks. - cmakeBuild() and autoconfBuild() are the two builds it picks from. The shell they run is the shell that ran before, with one line added: the build is a second sh, so it enters the directory the checkout left off in. withSccache now wraps the build alone rather than the checkout as well, which never needed the compile cache in its environment. The cleans of the checkout say -ffdx. A single -f skips an untracked directory that is a repository of its own, which is exactly what a submodule that has been dropped leaves behind: OMCompiler/3rdParty/sundials-5.4.0 has been sitting on the nodes since sundials became a submodule of its own name. The clean of the OpenModelica checkout keeps libraries/git, the reference file clones test.py fills GITREPOS with, which -ff would otherwise remove on every run. Co-Authored-By: Claude Opus 5 --- .CI/common.groovy | 257 +++++++++++++++++++++++++++------------------- 1 file changed, 153 insertions(+), 104 deletions(-) diff --git a/.CI/common.groovy b/.CI/common.groovy index 6a0fa42..da59106 100644 --- a/.CI/common.groovy +++ b/.CI/common.groovy @@ -190,6 +190,155 @@ def fmiSimulators(boolean omsimulator, boolean fmpy) { return simulators } +/** + * Checks out the sources of a run into OpenModelica/OMCompiler of the workspace: + * the clone if the node has none, the branch or the pull request ref, and the + * submodules. A `.nogit` in the cached build keeps the sources as they are, for + * a node the test was set up on by hand. + */ +def checkoutOMC(name, String checkoutRef, Closure runSh) { + runSh(""" + if test ! -d OpenModelica; then + git clone --recursive https://openmodelica.org/git-readonly/OpenModelica.git OpenModelica + fi + cd OpenModelica + git fetch + git reset --hard origin/master + # -ff, or the directory of a submodule that has been dropped stays for ever: + # git clean skips an untracked directory that is a repository of its own. + # libraries/git is the one such directory to keep, the reference file clones + # test.py fills GITREPOS with. + git clean -ffdx -e /libraries/git + + cd OMCompiler + + if ! test -f ~/saved_omc/${name}/.nogit; then + ${checkoutRef} + git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 + git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -ffdxq -e /git -e /svn" || exit 1 + git clean -ffdxq || exit 1 + git submodule status --recursive + fi + """) +} + +/** + * The shell that builds omc with cmake, in the build_cmake directory next to the + * source tree, and reports what the shared compile cache did for it. The target + * asking for a cmake build is what `cmakeFlags` comes from; the release flags + * every such build shares are here. + */ +def cmakeBuild(String cmakeFlags, buildJobs) { + // Run from OpenModelica/OMCompiler, one directory below the cmake source tree. + return sccachePreamble() + """ + cmake -S .. -B ../build_cmake + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="`pwd`/build" \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_Fortran_COMPILER=gfortran \ + -DCMAKE_C_FLAGS=-march=native \ + -DCMAKE_CXX_FLAGS=-march=native \ + -DOM_USE_CCACHE=OFF \ + -DOM_ENABLE_GUI_CLIENTS=OFF \ + -DOM_ENABLE_OMSIMULATOR=OFF \ + ${cmakeFlags} || exit 1 + if ! time cmake --build ../build_cmake --parallel ${buildJobs} --target install > log 2>&1; then + cat log + exit 1 + fi + build/bin/omc --version || exit 1 + sccache --show-stats || true + """ +} + +/** + * The shell that builds omc with autotools, which is what a target that does not + * ask for a cmake build gets. A failing C++ runtime only fails the build of the + * master target: the maintenance branches have been living with it. + */ +def autoconfBuild(name, buildJobs) { + return """ + autoreconf --install + ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omniORB + time make -j${buildJobs} clean + if ! time make -j${buildJobs} omc > log 2>&1; then + cat log + exit 1 + fi + if ! time make -j${buildJobs} runtimeCPPinstall > log 2>&1; then + cat log + if test "${name}" = "master"; then + exit 1 + else + echo "Ignoring failed C++ runtime" + fi + fi + """ +} + +/** + * Builds omc, or restores the build of the same sources from ~/saved_omc/. + * The hash of the checkout is the key of that cache, and a restored build that + * does not run here is built again: a cache filled on the node before the jobs + * moved into a container, or against an older image, matches by hash but holds a + * binary linked against libraries that are missing now. + * + * Target-specific `cmakeFlags` build omc with cmake and the shared release flags + * added here; empty is the autotools build every other target asks for. + */ +def buildOrRestoreOMC(name, String cmakeFlags, Closure runSh) { + // The build used to say -j9, the cores of the machine this file was written + // for in 2019, and -j16, the cores of the ryzen-5950x machines that replaced + // it - the commit that raised the others to 16 left the omc build at 9. Named + // buildJobs rather than jobs: that is this method's own parameter, how many + // models test.py tests at a time. + def buildJobs = numPhysicalCPU() + echo "Building omc with -j${buildJobs} on ${env.NODE_NAME}" + + def buildOMC = cmakeFlags ? cmakeBuild(cmakeFlags, buildJobs) : autoconfBuild(name, buildJobs) + + runSh(""" + cd OpenModelica/OMCompiler + export OPENMODELICAHOME="`pwd`/build" + + git rev-parse --verify HEAD > .newhash + echo New Hash: + cat .newhash + echo Old Hash: + cat ~/saved_omc/${name}/.githash || true + REBUILD="" + if cmp ~/saved_omc/${name}/.githash .newhash || test -f ~/saved_omc/${name}/.nogit; then + rsync -a --delete ~/saved_omc/${name}/ build/ || exit 1 + echo "Restoring cached OMC version: ${name}, `cat ~/saved_omc/${name}/.githash`" + # The hash says the sources match, not that the binary runs here: a cache + # filled on the node before the jobs moved into a container, or before the + # image changed, holds an omc linked against libraries that are missing now. + # Without this check the run dies much later, when test.py asks the restored + # binary for its version and gets exit code 127. + if ! build/bin/omc --version; then + if test -f ~/saved_omc/${name}/.nogit; then + echo "The cached omc of ${name} does not run in this environment, and .nogit forbids rebuilding it." + exit 1 + fi + echo "The cached omc of ${name} does not run in this environment; rebuilding it." + REBUILD=1 + fi + else + REBUILD=1 + fi + if test -n "\$REBUILD"; then + ${buildOMC} + rm -rf ~/saved_omc/${name}/ + mkdir -p ~/saved_omc/${name}/ + CMD="rsync -a --delete build/ \$HOME/saved_omc/${name}/" + echo \$CMD + \$CMD || exit 1 + cp .newhash ~/saved_omc/${name}/.githash + fi + """) +} + /** * Launches the test.py script with the given options. * @@ -308,7 +457,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, extrasimflags, testFlag git submodule sync --recursive || exit 1 git clean -ffdx || exit 1 git submodule update --init --recursive --force || exit 1 - git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 + git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -ffdxq -e /git -e /svn" || exit 1 cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=install/ cmake --build build/ --target install || exit 1 ./install/bin/OMSimulator --version || exit 1 @@ -367,51 +516,6 @@ def runRegressiontest(branch, name, extraFlags, omsHash, extrasimflags, testFlag git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 """ - // The build used to say -j9, the cores of the machine this file was written - // for in 2019, and -j16, the cores of the ryzen-5950x machines that replaced - // it - the commit that raised the others to 16 left the omc build at 9. Named - // buildJobs rather than jobs: that is this method's own parameter, how many - // models test.py tests at a time. - def buildJobs = numPhysicalCPU() - echo "Building omc with -j${buildJobs} on ${env.NODE_NAME}" - - def buildOMC - if (cmakeFlags) { - // Run from OpenModelica/OMCompiler, one directory below the cmake source tree. - buildOMC = sccachePreamble() + """ - cmake -S .. -B ../build_cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="`pwd`/build" \ - -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_Fortran_COMPILER=gfortran \ - -DCMAKE_C_FLAGS=-march=native -DCMAKE_CXX_FLAGS=-march=native \ - -DOM_USE_CCACHE=OFF -DOM_ENABLE_GUI_CLIENTS=OFF -DOM_ENABLE_OMSIMULATOR=OFF \ - ${cmakeFlags} || exit 1 - if ! time cmake --build ../build_cmake --parallel ${buildJobs} --target install > log 2>&1; then - cat log - exit 1 - fi - build/bin/omc --version || exit 1 - sccache --show-stats || true - """ - } else { - buildOMC = """ - autoreconf --install - ./configure --with-cppruntime --without-omc --disable-modelica3d CC=clang CXX=clang++ FC=gfortran CFLAGS='-O2 -march=native' --with-omniORB - time make -j${buildJobs} clean - if ! time make -j${buildJobs} omc > log 2>&1; then - cat log - exit 1 - fi - if ! time make -j${buildJobs} runtimeCPPinstall > log 2>&1; then - cat log - if test "${name}" = "master"; then - exit 1 - else - echo "Ignoring failed C++ runtime" - fi - fi - """ - } - sh ''' FREE=`df -k --output=avail "$PWD" | tail -n1` # df -k not df -h if test "$FREE" -lt 31457280; then # 30G = 30*1024*1024k @@ -422,67 +526,12 @@ def runRegressiontest(branch, name, extraFlags, omsHash, extrasimflags, testFlag sh 'killall omc || true' - def checkoutAndBuild = """ - if test ! -d OpenModelica; then - git clone --recursive https://openmodelica.org/git-readonly/OpenModelica.git OpenModelica - fi - cd OpenModelica - git fetch - git reset --hard origin/master - git clean -fdx - - cd OMCompiler - - if ! test -f ~/saved_omc/${name}/.nogit; then - ${checkoutRef} - git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 - git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 - git clean -fdxq || exit 1 - git submodule status --recursive - fi - - export OPENMODELICAHOME="`pwd`/build" - - git rev-parse --verify HEAD > .newhash - echo New Hash: - cat .newhash - echo Old Hash: - cat ~/saved_omc/${name}/.githash || true - REBUILD="" - if cmp ~/saved_omc/${name}/.githash .newhash || test -f ~/saved_omc/${name}/.nogit; then - rsync -a --delete ~/saved_omc/${name}/ build/ || exit 1 - echo "Restoring cached OMC version: ${name}, `cat ~/saved_omc/${name}/.githash`" - # The hash says the sources match, not that the binary runs here: a cache - # filled on the node before the jobs moved into a container, or before the - # image changed, holds an omc linked against libraries that are missing now. - # Without this check the run dies much later, when test.py asks the restored - # binary for its version and gets exit code 127. - if ! build/bin/omc --version; then - if test -f ~/saved_omc/${name}/.nogit; then - echo "The cached omc of ${name} does not run in this environment, and .nogit forbids rebuilding it." - exit 1 - fi - echo "The cached omc of ${name} does not run in this environment; rebuilding it." - REBUILD=1 - fi - else - REBUILD=1 - fi - if test -n "\$REBUILD"; then - ${buildOMC} - rm -rf ~/saved_omc/${name}/ - mkdir -p ~/saved_omc/${name}/ - CMD="rsync -a --delete build/ \$HOME/saved_omc/${name}/" - echo \$CMD - \$CMD || exit 1 - cp .newhash ~/saved_omc/${name}/.githash - fi - """ + checkoutOMC(name, checkoutRef, runSh) if (cmakeFlags) { - withSccache { runSh(checkoutAndBuild) } + withSccache { buildOrRestoreOMC(name, cmakeFlags, runSh) } } else { - runSh(checkoutAndBuild) + buildOrRestoreOMC(name, cmakeFlags, runSh) } sh """