.gitlab-ci.yml 9.04 KB
Newer Older
Martin Bauer's avatar
Martin Bauer committed
1
stages:
2
  - pretest
Martin Bauer's avatar
Martin Bauer committed
3
4
5
6
7
8
9
10
  - test
  - deploy


# --------------------------  Tests ------------------------------------------------------------------------------------

# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
11
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
12
13
14
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
15
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
16
  script:
17
    - env
18
    - pip list
Martin Bauer's avatar
Martin Bauer committed
19
20
21
22
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
23
24
    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
    - python3 -m coverage xml
Martin Bauer's avatar
Martin Bauer committed
25
26
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
27
    - cuda11
Martin Bauer's avatar
Martin Bauer committed
28
29
30
31
32
    - AVX
  artifacts:
    when: always
    paths:
      - coverage_report
33
      - test-report
34
35
36
    reports:
      cobertura: coverage.xml
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# pipeline with latest python version
latest-python:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
  script:
    - env
    - pip list
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - py.test -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
  tags:
    - docker
    - AVX
  artifacts:
    when: always
    reports:
      junit: report.xml

Martin Bauer's avatar
Martin Bauer committed
61
62
63
64
65
66
# Nightly test  - runs "long run" jobs only
test-longrun:
  stage: test
  only:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
67
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
68
69
70
71
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
72
    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --html test-report/index.html --junitxml=report.xml
Martin Bauer's avatar
Martin Bauer committed
73
74
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
75
    - cuda11
Martin Bauer's avatar
Martin Bauer committed
76
77
    - AVX
  artifacts:
78
    when: always
Martin Bauer's avatar
Martin Bauer committed
79
80
    paths:
      - coverage_report
81
      - test-report
82
83
    reports:
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
84
85
86
87
88
89
90
91
92
93

# Minimal tests in windows environment
minimal-windows:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  tags:
    - win
  script:
94
    - export NUM_CORES=$(nproc --all)
Martin Bauer's avatar
Martin Bauer committed
95
    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
96
97
    - source activate pystencils
    - pip list
Martin Bauer's avatar
Martin Bauer committed
98
    - python -c "import numpy"
99
    - py.test -v -m "not (notebook or longrun)"
Martin Bauer's avatar
Martin Bauer committed
100

101
ubuntu:
Martin Bauer's avatar
Martin Bauer committed
102
103
104
105
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
106
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
107
108
109
110
  before_script:
    - apt-get -y remove python3-sympy
    - ln -s /usr/include/locale.h /usr/include/xlocale.h
    - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
Martin Bauer's avatar
Martin Bauer committed
111
  script:
112
    - export NUM_CORES=$(nproc --all)
113
114
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
Michael Kuron's avatar
Michael Kuron committed
115
    - sed -i 's/--doctest-modules //g' pytest.ini
116
117
118
    - env
    - pip3 list
    - pytest-3 -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
Martin Bauer's avatar
Martin Bauer committed
119
120
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
121
    - cuda11
Stephan Seitz's avatar
Stephan Seitz committed
122
    - AVX
123
124
125
126
  artifacts:
    when: always
    reports:
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
127

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
.multiarch_template:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  before_script: &multiarch_before_script
    - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"
    - sed -i '/^fail_under.*/d' pytest.ini
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - sed -i 's/--doctest-modules //g' pytest.ini
    - env
    - pip3 list
    - pytest-3 -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --junitxml=report.xml pystencils_tests/test_*vec*.py pystencils_tests/test_random.py
    - python3 -m coverage xml
  tags:
    - docker
  artifacts:
    when: always
    paths:
      - coverage_report
    reports:
      cobertura: coverage.xml
      junit: report.xml

Michael Kuron's avatar
Michael Kuron committed
155
arm64v8:
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  variables:
    PYSTENCILS_SIMD: "neon"
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv8-a/g ~/.config/pystencils/config.json

ppc64le:
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ppc64le
  variables:
    PYSTENCILS_SIMD: "vsx"
  before_script:
    - *multiarch_before_script
    - sed -i s/mcpu=native/mcpu=power8/g ~/.config/pystencils/config.json

Michael Kuron's avatar
Michael Kuron committed
173
174
175
176
177
178
179
180
181
arm64v9:
  # Compiler support for SVE is still pretty rough: GCC 10+11 produce incorrect code for fixed-width vectors,
  # while Clang 12 produces memory-corrupting heisenbugs unless we enable the address sanitizer.
  # In the RNG tests, GCC 10+11 produce an internal compiler error.
  # The memory corruption seems to only happen with qemu-user, not with qemu-system.
  # Once the compilers and QEMU have improved, this job should be cleaned up to match the others.
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  variables:
Michael Kuron's avatar
Michael Kuron committed
182
    PYSTENCILS_SIMD: "sve256,sve512,sve"
Michael Kuron's avatar
Michael Kuron committed
183
184
185
186
187
188
189
    ASAN_OPTIONS: detect_leaks=0
    LD_PRELOAD: /usr/lib/aarch64-linux-gnu/libasan.so.6
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv8-a+sve/g ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json

Michael Kuron's avatar
Michael Kuron committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
riscv64:
  # The RISC-V vector extension is still experimental and needs special compiler flags.
  # Once they are officially released, this job should be cleaned up to match the others.
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/riscv64
  variables:
    PYSTENCILS_SIMD: "rvv"
    QEMU_CPU: "rv64,x-v=true"
  before_script:
    - *multiarch_before_script
    - sed -i 's/march=native/march=rv64imfdv0p10 -menable-experimental-extensions/g' ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json
    - sed -i 's/fopenmp/fopenmp=libgomp -I\/usr\/include\/riscv64-linux-gnu/g' ~/.config/pystencils/config.json

Martin Bauer's avatar
Martin Bauer committed
204
minimal-conda:
205
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
206
207
208
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
209
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
Martin Bauer's avatar
Martin Bauer committed
210
211
212
213
  script:
    - python setup.py quicktest
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
214
    - cuda
Martin Bauer's avatar
Martin Bauer committed
215
216


Stephan Seitz's avatar
Stephan Seitz committed
217
218
219
220
221
222
223
224
225
226
227
228
minimal-sympy-master:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
  script:
    - python -m pip install --upgrade git+https://github.com/sympy/sympy.git
    - python setup.py quicktest
  allow_failure: true
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
229
    - cuda
Stephan Seitz's avatar
Stephan Seitz committed
230

231
232
233
234
235

pycodegen-integration:
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  stage: test
  when: manual
Markus Holzer's avatar
Markus Holzer committed
236
  allow_failure: true
237
238
239
240
241
242
243
244
  script:
    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@i10git.cs.fau.de/pycodegen/pycodegen.git
    - cd pycodegen
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git submodule foreach git fetch origin   # compare the latest master version!
    - git submodule foreach git reset --hard origin/master
    - cd pystencils
Stephan Seitz's avatar
Stephan Seitz committed
245
246
    - git remote add test $CI_REPOSITORY_URL
    - git fetch test
247
248
    - git reset --hard $CI_COMMIT_SHA
    - cd ..
Markus Holzer's avatar
Markus Holzer committed
249
250
251
    - pip install -e pystencils/
    - pip install -e lbmpy/
    - pip install -e pygrandchem/
252
253
254
255
256
    - ./install_walberla.sh
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - cd pystencils
257
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
258
    - cd ../lbmpy
259
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
260
    - cd ../pygrandchem
261
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
262
    - cd ../walberla/build/
263
    - make CodegenJacobiCPU CodegenJacobiGPU CodegenPoissonCPU CodegenPoissonGPU MicroBenchmarkGpuLbm LbCodeGenerationExample UniformGridBenchmarkGPU_trt UniformGridBenchmarkGPU_entropic_kbc_n4 FluctuatingMRT
Markus Holzer's avatar
Markus Holzer committed
264
265
266
267
    - cd apps/benchmarks/UniformGridGPU
    - make -j $NUM_CORES
    - cd ../UniformGridGenerated
    - make -j $NUM_CORES
268
269
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
270
    - cuda11
271
    - AVX
272
273
274
  artifacts:
    when: always
    reports:
275
      junit: pycodegen/*/report.xml
276

Martin Bauer's avatar
Martin Bauer committed
277
278
279
280
# -------------------- Linter & Documentation --------------------------------------------------------------------------


flake8-lint:
281
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
282
283
284
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
285
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
286
287
288
289
290
291
292
293
  script:
    - flake8 pystencils
  tags:
    - docker


build-documentation:
  stage: test
294
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
295
296
297
  script:
    - export PYTHONPATH=`pwd`
    - mkdir html_doc
Markus Holzer's avatar
Markus Holzer committed
298
    - sphinx-build -b html doc  html_doc
Martin Bauer's avatar
Martin Bauer committed
299
300
301
302
303
304
305
306
307
    - sphinx-build -W -b html doc  html_doc
  tags:
    - docker
  artifacts:
    paths:
      - html_doc


pages:
308
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
309
310
311
312
313
314
315
316
317
318
319
  stage: deploy
  script:
    - ls -l
    - mv coverage_report html_doc
    - mv html_doc public  # folder has to be named "public" for gitlab to publish it
  artifacts:
    paths:
      - public
  tags:
    - docker
  only:
320
    - master@pycodegen/pystencils