.gitlab-ci.yml 9.08 KB
Newer Older
Martin Bauer's avatar
Martin Bauer committed
1
stages:
2
  - pretest
Martin Bauer's avatar
Martin Bauer committed
3
4
5
6
7
8
9
10
  - test
  - deploy


# --------------------------  Tests ------------------------------------------------------------------------------------

# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
11
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
12
13
14
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
15
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
16
  script:
17
    - env
18
    - pip list
Martin Bauer's avatar
Martin Bauer committed
19
20
21
22
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
23
24
    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
    - python3 -m coverage xml
Martin Bauer's avatar
Martin Bauer committed
25
26
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
27
    - cuda11
Martin Bauer's avatar
Martin Bauer committed
28
29
30
31
32
    - AVX
  artifacts:
    when: always
    paths:
      - coverage_report
33
      - test-report
34
35
36
    reports:
      cobertura: coverage.xml
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# pipeline with latest python version
latest-python:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
  script:
    - env
    - pip list
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - py.test -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
  tags:
    - docker
    - AVX
  artifacts:
    when: always
    reports:
      junit: report.xml

Martin Bauer's avatar
Martin Bauer committed
61
62
63
64
65
66
# Nightly test  - runs "long run" jobs only
test-longrun:
  stage: test
  only:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
67
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
68
69
70
71
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
72
    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --html test-report/index.html --junitxml=report.xml
Martin Bauer's avatar
Martin Bauer committed
73
74
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
75
    - cuda11
Martin Bauer's avatar
Martin Bauer committed
76
77
    - AVX
  artifacts:
78
    when: always
Martin Bauer's avatar
Martin Bauer committed
79
80
    paths:
      - coverage_report
81
      - test-report
82
83
    reports:
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
84
85
86
87
88
89
90
91
92
93

# Minimal tests in windows environment
minimal-windows:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  tags:
    - win
  script:
94
    - export NUM_CORES=$(nproc --all)
Martin Bauer's avatar
Martin Bauer committed
95
    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
96
97
    - source activate pystencils
    - pip list
Martin Bauer's avatar
Martin Bauer committed
98
    - python -c "import numpy"
99
    - py.test -v -m "not (notebook or longrun)"
Martin Bauer's avatar
Martin Bauer committed
100

101
ubuntu:
Martin Bauer's avatar
Martin Bauer committed
102
103
104
105
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
106
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
107
108
109
110
  before_script:
    - apt-get -y remove python3-sympy
    - ln -s /usr/include/locale.h /usr/include/xlocale.h
    - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
Martin Bauer's avatar
Martin Bauer committed
111
  script:
112
    - export NUM_CORES=$(nproc --all)
113
114
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
Michael Kuron's avatar
Michael Kuron committed
115
    - sed -i 's/--doctest-modules //g' pytest.ini
116
117
118
    - env
    - pip3 list
    - pytest-3 -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
Martin Bauer's avatar
Martin Bauer committed
119
120
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
121
    - cuda11
Stephan Seitz's avatar
Stephan Seitz committed
122
    - AVX
123
124
125
126
  artifacts:
    when: always
    reports:
      junit: report.xml
Martin Bauer's avatar
Martin Bauer committed
127

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
.multiarch_template:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  before_script: &multiarch_before_script
    - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"
    - sed -i '/^fail_under.*/d' pytest.ini
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - sed -i 's/--doctest-modules //g' pytest.ini
    - env
    - pip3 list
    - pytest-3 -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --junitxml=report.xml pystencils_tests/test_*vec*.py pystencils_tests/test_random.py
    - python3 -m coverage xml
  tags:
    - docker
147
    - AVX
148
149
150
151
152
153
154
155
  artifacts:
    when: always
    paths:
      - coverage_report
    reports:
      cobertura: coverage.xml
      junit: report.xml

Michael Kuron's avatar
Michael Kuron committed
156
arm64v8:
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  variables:
    PYSTENCILS_SIMD: "neon"
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv8-a/g ~/.config/pystencils/config.json

ppc64le:
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ppc64le
  variables:
    PYSTENCILS_SIMD: "vsx"
  before_script:
    - *multiarch_before_script
    - sed -i s/mcpu=native/mcpu=power8/g ~/.config/pystencils/config.json

Michael Kuron's avatar
Michael Kuron committed
174
175
176
177
178
179
180
181
182
arm64v9:
  # Compiler support for SVE is still pretty rough: GCC 10+11 produce incorrect code for fixed-width vectors,
  # while Clang 12 produces memory-corrupting heisenbugs unless we enable the address sanitizer.
  # In the RNG tests, GCC 10+11 produce an internal compiler error.
  # The memory corruption seems to only happen with qemu-user, not with qemu-system.
  # Once the compilers and QEMU have improved, this job should be cleaned up to match the others.
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  variables:
Michael Kuron's avatar
Michael Kuron committed
183
    PYSTENCILS_SIMD: "sve256,sve512,sve"
Michael Kuron's avatar
Michael Kuron committed
184
185
186
187
188
189
190
    ASAN_OPTIONS: detect_leaks=0
    LD_PRELOAD: /usr/lib/aarch64-linux-gnu/libasan.so.6
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv8-a+sve/g ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json

Michael Kuron's avatar
Michael Kuron committed
191
192
193
194
195
196
197
198
199
200
201
202
203
204
riscv64:
  # The RISC-V vector extension is still experimental and needs special compiler flags.
  # Once they are officially released, this job should be cleaned up to match the others.
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/riscv64
  variables:
    PYSTENCILS_SIMD: "rvv"
    QEMU_CPU: "rv64,x-v=true"
  before_script:
    - *multiarch_before_script
    - sed -i 's/march=native/march=rv64imfdv0p10 -menable-experimental-extensions/g' ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json
    - sed -i 's/fopenmp/fopenmp=libgomp -I\/usr\/include\/riscv64-linux-gnu/g' ~/.config/pystencils/config.json

Martin Bauer's avatar
Martin Bauer committed
205
minimal-conda:
206
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
207
208
209
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
210
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
Martin Bauer's avatar
Martin Bauer committed
211
212
213
214
  script:
    - python setup.py quicktest
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
215
    - cuda
Martin Bauer's avatar
Martin Bauer committed
216
217


Stephan Seitz's avatar
Stephan Seitz committed
218
219
220
221
222
223
224
225
226
227
228
229
minimal-sympy-master:
  stage: test
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
  script:
    - python -m pip install --upgrade git+https://github.com/sympy/sympy.git
    - python setup.py quicktest
  allow_failure: true
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
230
    - cuda
Stephan Seitz's avatar
Stephan Seitz committed
231

232
233
234
235
236

pycodegen-integration:
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  stage: test
  when: manual
Markus Holzer's avatar
Markus Holzer committed
237
  allow_failure: true
238
239
240
241
242
243
244
245
  script:
    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@i10git.cs.fau.de/pycodegen/pycodegen.git
    - cd pycodegen
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git submodule foreach git fetch origin   # compare the latest master version!
    - git submodule foreach git reset --hard origin/master
    - cd pystencils
Stephan Seitz's avatar
Stephan Seitz committed
246
247
    - git remote add test $CI_REPOSITORY_URL
    - git fetch test
248
249
    - git reset --hard $CI_COMMIT_SHA
    - cd ..
Markus Holzer's avatar
Markus Holzer committed
250
251
252
    - pip install -e pystencils/
    - pip install -e lbmpy/
    - pip install -e pygrandchem/
253
254
255
256
257
    - ./install_walberla.sh
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - cd pystencils
258
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
259
    - cd ../lbmpy
260
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
261
    - cd ../pygrandchem
262
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
263
    - cd ../walberla/build/
Markus Holzer's avatar
Markus Holzer committed
264
265
    - make -j $NUM_CORES CodegenJacobiCPU CodegenJacobiGPU CodegenPoissonCPU CodegenPoissonGPU MicroBenchmarkGpuLbm LbCodeGenerationExample
    - make -j $NUM_CORES multiphaseCPU multiphaseGPU FluctuatingMRT FlowAroundSphereCodeGen
Markus Holzer's avatar
Markus Holzer committed
266
267
268
269
    - cd apps/benchmarks/UniformGridGPU
    - make -j $NUM_CORES
    - cd ../UniformGridGenerated
    - make -j $NUM_CORES
270
271
  tags:
    - docker
Markus Holzer's avatar
Markus Holzer committed
272
    - cuda11
273
    - AVX
274
275
276
  artifacts:
    when: always
    reports:
277
      junit: pycodegen/*/report.xml
278

Martin Bauer's avatar
Martin Bauer committed
279
280
281
282
# -------------------- Linter & Documentation --------------------------------------------------------------------------


flake8-lint:
283
  stage: pretest
Martin Bauer's avatar
Martin Bauer committed
284
285
286
  except:
    variables:
      - $ENABLE_NIGHTLY_BUILDS
287
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
288
289
290
291
292
293
294
295
  script:
    - flake8 pystencils
  tags:
    - docker


build-documentation:
  stage: test
296
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
297
298
299
  script:
    - export PYTHONPATH=`pwd`
    - mkdir html_doc
Markus Holzer's avatar
Markus Holzer committed
300
    - sphinx-build -b html doc  html_doc
Martin Bauer's avatar
Martin Bauer committed
301
302
303
304
305
306
307
308
309
    - sphinx-build -W -b html doc  html_doc
  tags:
    - docker
  artifacts:
    paths:
      - html_doc


pages:
310
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
Martin Bauer's avatar
Martin Bauer committed
311
312
313
314
315
316
317
318
319
320
321
  stage: deploy
  script:
    - ls -l
    - mv coverage_report html_doc
    - mv html_doc public  # folder has to be named "public" for gitlab to publish it
  artifacts:
    paths:
      - public
  tags:
    - docker
  only:
322
    - master@pycodegen/pystencils