1---
2# This node will be ignored while unrolling tests
3# as node name starts with a dot "."
4.this_is_a_job:
5 # A node can reference a "group", a profile-wide template to propagate multiple
6 # TE keys to different YAML files at once
7 group: "GRPSERIAL"
8 # A tag is a label to classify jobs when displaying results
9 tag: ["a", "b"]
10 # Build rules => how to build the benchmark
11 build:
12 # Files used to build this node
13 # it can be a source file, a makefile...
14 files: "@SRCPATH@/*.c"
15 # ===== FOR ANY BUILD SYSTEM
16 # args: arguments forwarded to the build system
17 # envs: environment variables exported to the target shell.
18 # Only one build system is expected to be used at once among the following:
19
20 autotools:
21 args: ['--disable-bootstrap']
22 envs: []
23 cmake:
24 # "vars" is a shortcut to "args: ["-D<var>"]"
25 vars: ['CMAKE_VERBOSE_MAKEFILE=ON']
26 args: []
27 envs: []
28 make:
29 # default makefile target to invoke
30 target: "all"
31 # a list of macros can be used as placeholder to be replaced when
32 # unrolling. A list is available at the end of this file.
33 args: ['MPICC=@COMPILER_CC@']
34 envs: []
35 sources:
36 # program binary (if needed)
37 binary: "a.out"
38 # extra cflags
39 cflags: "extra cflags"
40 # extra ldflags
41 ldflags: "extra ldflags"
42 envs: []
43 # force explicit language if PCVS has trouble to autodetect
44 lang: ['cc', 'cxx', 'fc']
45 custom:
46 # Really complex build scenario PCVS may not be able to handle
47 program: "myscript.sh"
48 envs: ["VAR1=value", "VAR2=value"]
49 validate:
50 # same as validate node bellow but to validate the build process
51 # instead of the run.
52 [...]
53 # dependency scheme. This can be:
54 # a fully expanded test name
55 # a TE node (like ".this_is_a_job" for the current node).
56 # A circular detection is run.
57 depends_on: ["this_is_another_test"]
58 # directory where program should be built
59 cwd: "dir/to/build"
60 # variants required from profiles to run this job
61 # IF these variants are not satisfied, jobs will be discarded.
62 # it indicates not support for a specific variant.
63 # There is no preset variants, all of them can be declared by the profile.
64 # It can be found under compiler.$LANG.variants
65 variants:
66 - openmp
67 - accel
68 - mpi
69 # PM interaction, allow loading package/module BEFORE running jobs
70 # This load will be done for every job, it may create overhead.
71 # currently, only spack recipes & module will be allowed.
72 # If a spack recipe is not available, it will be installed first.
73 package_manager:
74 spack:
75 - protobuf@3.1.1
76 - gcc@7.3.0
77 module:
78 - protobuf
79 - gnu/gcc/7.3.0
80
81 run: &run_part
82 # path to program (defaults to build.sources.binary if not set)
83 program: "./a.out"
84 # Scale the benchmark to build test scenarios.
85 iterate:
86 # subnodes are parsed relatively to the profile configuration.
87 # There are expected to be declared to profiles, under "criterion"
88 # only intersected values with profiles will be kept.
89 n_mpi:
90 values: [2, 4]
91 n_omp:
92 values: [1, 2]
93 # Special key to declare program-scope parameter matrix.
94 # One can use as many parameters to run the given program.
95 # criterion name will be given by the key
96 # criterion values will be given by the "values" field
97 # criterion label (used to generate test-name) will be given by "subtitle"
98 # parameter position will be given by "type": "argument" or "environment"
99 # numeric attributes allow using sequence to generate values.
100 # for instance, to generate only power of 2 values:
101 # => values: [{from: 0; to: 100, op: "mul": of: 2}]
102 program:
103 give_it_a_name:
104 numeric: true
105 # false send the argument to the wrapper of the program
106 # true (default) send the argument to the program itself
107 local: true
108 type: "argument"
109 values: ["-iter 1000", "-fast"]
110 subtitle: "lol"
111 # directory where program should be built
112 cwd: "dir/to/build"
113 # dependency scheme, same rules as build.depends_on.
114 depends_on: ["this_is_another_run_test_in_the_same_file"]
115 # PM interaction, same as build.package_manager
116 package_manager:
117 spack:
118 - protobuf@3.1.1
119 - gcc@7.3.0
120 module:
121 - protobuf
122 - gnu/gcc/7.3.0
123 # How to validate jobs generated from this specification ?
124 # each of these subkeys are cumulative, all of them has to be true to
125 # mark the job as a success
126 validate:
127 # expected return code
128 expect_exit: 0
129 # time limit
130 time:
131 # max execution time the job should not exceed to succeed
132 mean: 10.0
133 # time measurement may vary. Some tolerance is allowed (only upper-bound)
134 tolerance: 2.0
135 # After this time, job will be automatically stopped and marked as failed
136 # note that PCVS cannot be able to handle every job kill mechanism.
137 # Our approach is to terminate the process which spawned the test on
138 # resources. With some batch-managers, it may lead to leaks.
139 hard_timeout: 40
140 # Mark the job as failed without killing it.
141 # This allow you to run the test for code correctness on a slow machine
142 # without testing for speed.
143 soft_timeout: 20
144 # Output formatting compliance
145 # It is based on rule definition as regex. These regexs has to be matched (or
146 # not) to pass.
147 match:
148 # rule name
149 label:
150 # the regex
151 expr: '^\d+(\.\d+) received$'
152 # Is the pattern allowed or forbidden in the output.
153 # True: The regex MUST match to success
154 # False: The regex MUST NOT match to success
155 # default: True
156 expect: true|false
157 label2: 'Total Elapsed: \d+\.\d+ sec.$'
158 # Analysis method
159 # based on dedicated Python module (builtin or user-made) to be run on every
160 # jobs to select the final status
161 # use-case: mark job status depending on its execution history over past
162 # runs.
163 analysis:
164 method: "<method>"
165 # Generic scenario, not handled by PCVS.
166 # Job info & output are provided to the script, returning 0 in case of
167 # success, non-zero otherwise.
168 script:
169 path: "/path/to/script"
170
171 # Post execution rules.
172 # artifacts can be stored to the final results to store job-specific data.
173 # Currently, artifacts only supports filesystem
174 artifact:
175 # relative to $BUILDPATH
176 obj1: "./path/1"
177 obj2: "./path/2"
178
179 # From the output, extract data to be stored as results.
180 # A regex is used to capture the metric from the standard output and is stored
181 # to the final result JSON file for later reference.
182 metrics:
183 # metric name
184 metric1:
185 # regex to match
186 key: "regex"
187 metric2:
188 key: "regex"
189 attributes:
190 # if multiple occurrences of the regex are found, only keep the first
191 # match. (default: false)
192 unique: true
193
194 # A list of properties to enable specific features.
195 # These attributes are common to build/run configurations
196 # if a different set of attributes are required for build & run a single TE,
197 # feel free to create TWO nodes, with a dep on "build".
198 attributes:
199 # control if the current benchmark program should be prefixed by
200 # the runtime.program command (default: true)
201 # True: ./a.out -> mpirun ./a.out
202 # False: ./a.out -> ./a.out
203 command_wrap: true
204 # controls if the relative path defined by this TE should be converted to
205 # absolute path to avoid PATH resolution. (default: true)
206 # True: ./a.out -> @BUILDIR@/./a.out
207 # False: spack install -> spack install
208 path_resolution: true
209 # duplicates inputs defined by this TE for any generated test.
210 # For instance, if the current build directory provide a non-shareable
211 # resource, it may run into trouble when running concurrent tests from this
212 # specification. A dedicated directory is built for each subtest.
213 # (default: false)
214 copy_input: false
215 # duplicates outputs defined by this TE for any generated test.
216 # This is the opposite of copy_input, to allow generating concurrent artifacts
217 # from this single specification (default: false)
218 copy_output: false
219
220
221#########################################################
222
223# depicts an inheritance mechanism.
224# a key is defined as &key, as put on the "run" node above.
225# It can then be referred with "*key". Any subkeys are copied
226# to under the target node.
227# This can be used to factor keys among multiples job descriptions
228real_test:
229 build:
230 make:
231 target: all
232 run:
233 <<: *run_part