Skip to content

Ci debug distributions #3204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Ci debug distributions #3204

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1e1b6e
Use Jenkins executor-local PARALLEL value
WardBrian Jun 11, 2025
52a5a9d
Request some more expensive tests always end up on 8core systems
WardBrian Jun 11, 2025
2417eb6
debug
WardBrian Jun 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
328 changes: 6 additions & 322 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ def runTests(String testPath, boolean jumbo = false) {
sh "cat make/local"
sh "make print-compiler-flags"
if (jumbo && !params.disableJumbo) {
sh "python3 runTests.py -j${env.PARALLEL} ${testPath} --jumbo --debug"
sh "python3 runTests.py -j${PARALLEL} ${testPath} --jumbo --debug"
} else {
sh "python3 runTests.py -j${env.PARALLEL} ${testPath}"
sh "python3 runTests.py -j${PARALLEL} ${testPath}"
}
}
finally { junit 'test/**/*.xml' }
Expand Down Expand Up @@ -61,7 +61,6 @@ pipeline {
OPENCL_PLATFORM_ID = 1
OPENCL_PLATFORM_ID_CPU = 0
OPENCL_PLATFORM_ID_GPU = 0
PARALLEL = 4
GIT_AUTHOR_NAME = 'Stan Jenkins'
GIT_AUTHOR_EMAIL = '[email protected]'
GIT_COMMITTER_NAME = 'Stan Jenkins'
Expand Down Expand Up @@ -173,305 +172,7 @@ pipeline {
post { always { retry(3) { deleteDir() } } }
}

stage('Quick tests') {
when {
expression {
!skipRemainingStages
}
}
failFast true
parallel {
stage('Headers check') {
when {
expression {
!skipRemainingStages
}
}
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
}
}

steps {
retry(1){
unstash 'MathSetup'
sh "echo CXX=${CLANG_CXX} -Werror > make/local"
sh "make -j${PARALLEL} test-headers"
}
}
post { always { deleteDir() } }
}
stage('Run changed unit tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
args '--cap-add SYS_PTRACE'
}
}
when {
allOf {
expression {
env.BRANCH_NAME ==~ /PR-\d+/
}
expression {
!skipRemainingStages
}
}
}
steps {
retry(3) { checkout scm }

sh "echo CXXFLAGS += -fsanitize=address >> make/local"
sh "./runTests.py -j${PARALLEL} --changed --debug"

}
post { always { retry(3) { deleteDir() } } }
}
}
}

stage('Full Unit Tests') {
when {
expression {
!skipRemainingStages
}
}
failFast true
parallel {
stage('Rev/Fwd Unit Tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
args '--cap-add SYS_PTRACE'
}
}
when {
expression {
!skipRemainingStages
}
}
steps {
unstash 'MathSetup'
sh "echo CXXFLAGS += -fsanitize=address >> make/local"

script {
if (!(params.optimizeUnitTests || isBranch('develop') || isBranch('master'))) {
sh "echo O=0 >> make/local"
}

runTests("test/unit/math/rev")
runTests("test/unit/math/fwd")
}
}
post { always { retry(3) { deleteDir() } } }
}
stage('Mix Unit Tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
args '--cap-add SYS_PTRACE'
}
}
when {
expression {
!skipRemainingStages
}
}
steps {
unstash 'MathSetup'
sh "echo CXXFLAGS += -fsanitize=address >> make/local"
script {
if (!(params.optimizeUnitTests || isBranch('develop') || isBranch('master'))) {
sh "echo O=1 >> make/local"
}
runTests("test/unit/math/mix", true)
}
}
post { always { retry(3) { deleteDir() } } }
}
stage('Prim Unit Tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
args '--cap-add SYS_PTRACE'
}
}
when {
expression {
!skipRemainingStages
}
}
steps {
unstash 'MathSetup'
sh "echo CXXFLAGS += -fsanitize=address >> make/local"
script {
if (!(params.optimizeUnitTests || isBranch('develop') || isBranch('master'))) {
sh "echo O=0 >> make/local"
}
runTests("test/unit/*_test.cpp", false)
runTests("test/unit/math/*_test.cpp", false)
runTests("test/unit/math/prim", true)
runTests("test/unit/math/memory", false)
}
}
post { always { retry(3) { deleteDir() } } }
}
stage('Laplace Unit Tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
args '--cap-add SYS_PTRACE'
}
}
when {
expression {
!skipRemainingStages
}
}
steps {
unstash 'MathSetup'
sh "echo CXXFLAGS += -march=native -mtune=native >> make/local"
sh "echo O=3 >> make/local"
script {
if (params.optimizeUnitTests || isBranch('develop') || isBranch('master')) {
sh "echo CXXFLAGS += -fsanitize=address >> make/local"
}
runTests("test/unit/math/laplace/*_test.cpp", false)
}
}
post { always { retry(3) { deleteDir() } } }
}
stage('OpenCL GPU tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'v100'
args '--gpus 1'
}
}
steps {
script {
unstash 'MathSetup'
sh """
echo CXX=${CLANG_CXX} -Werror > make/local
echo STAN_OPENCL=true >> make/local
echo OPENCL_PLATFORM_ID=${OPENCL_PLATFORM_ID_GPU} >> make/local
echo OPENCL_DEVICE_ID=${OPENCL_DEVICE_ID_GPU} >> make/local
"""
if (!(params.optimizeUnitTests || isBranch('develop') || isBranch('master'))) {
sh "echo O=1 >> make/local"
}
runTests("test/unit/math/opencl", false) // TODO(bward): try to enable
runTests("test/unit/multiple_translation_units_test.cpp")
}
}
post { always { retry(3) { deleteDir() } } }
}
}
}
stage('Always-run tests') {
when {
expression {
!skipRemainingStages
}
}
failFast true
parallel {
stage('MPI tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
}
}
steps {
unstash 'MathSetup'
sh """
echo CXX=${MPICXX} > make/local
echo CXX_TYPE=gcc >> make/local
echo STAN_MPI=true >> make/local
"""
runTests("test/unit/math/prim/functor")
runTests("test/unit/math/rev/functor")
}
post { always { retry(3) { deleteDir() } } }
}
stage('Expressions test') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
}
}
steps {
unstash 'MathSetup'
script {
sh "echo O=0 > make/local"
sh "echo CXX=${CLANG_CXX} -Werror >> make/local"
sh "python ./test/code_generator_test.py"
sh "python ./test/signature_parser_test.py"
sh "python ./test/statement_types_test.py"
sh "python ./test/varmat_compatibility_summary_test.py"
sh "python ./test/varmat_compatibility_test.py"
withEnv(['PATH+TBB=./lib/tbb']) {
sh "python ./test/expressions/test_expression_testing_framework.py"
sh "cat make/local"
try { sh "./runTests.py -j${PARALLEL} test/expressions" }
finally { junit 'test/**/*.xml' }
}
sh "make clean-all"
sh "echo STAN_THREADS=true >> make/local"
withEnv(['PATH+TBB=./lib/tbb']) {
try {
sh "cat make/local"
sh "./runTests.py -j${PARALLEL} test/expressions --only-functions reduce_sum map_rect"
}
finally { junit 'test/**/*.xml' }
}
}
}
post { always { deleteDir() } }
}

stage('Threading tests') {
agent {
docker {
image 'stanorg/ci:gpu-cpp17'
label 'linux'
}
}
steps {
script {
retry(1){
unstash 'MathSetup'
sh "echo CXX=${CLANG_CXX} -Werror > make/local"
sh "echo STAN_THREADS=true >> make/local"
sh "export STAN_NUM_THREADS=4"
if (isBranch('develop') || isBranch('master')) {
runTests("test/unit")
sh "find . -name *_test.xml | xargs rm"
} else {
runTests("test/unit -f thread")
sh "find . -name *_test.xml | xargs rm"
runTests("test/unit -f map_rect")
sh "find . -name *_test.xml | xargs rm"
runTests("test/unit -f reduce_sum")
}
}
}
}
post { always { retry(3) { deleteDir() } } }
}
}
}


stage ('Discover changed distribution tests') {
when {
expression {
Expand All @@ -490,7 +191,7 @@ pipeline {
if (params.runAllDistributions || isBranch('develop') || isBranch('master')) {
changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py --pretend-all", returnStdout:true).trim().readLines()
} else {
changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py", returnStdout:true).trim().readLines()
changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py --pretend-all", returnStdout:true).trim().readLines()
}
}
}
Expand All @@ -507,7 +208,7 @@ pipeline {
}
}
}
agent { label 'linux && docker' }
agent { label 'linux && docker && 8core' }
steps {
script {
def tests = [:]
Expand Down Expand Up @@ -547,24 +248,7 @@ pipeline {
}
}

stage('Upstream tests') {
agent { label 'linux' }
when {
allOf {
expression {
env.BRANCH_NAME ==~ /PR-\d+/
}
expression {
!skipRemainingStages
}
}
}
steps {
build(job: "Stan/Stan/${stan_pr()}",
parameters: [string(name: 'math_pr', value: env.BRANCH_NAME),
string(name: 'cmdstan_pr', value: cmdstan_pr())])
}
}


stage('Upload doxygen') {
agent {
Expand Down
Loading