Input Examples

Test nodes: The complete list

  1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/te-scheme.yml
  2---
  3# This node will be ignored while unrolling tests
  4# as node name starts with a dot "."
  5.this_is_a_job:
  6  # A node can reference a "group", a profile-wide template to propagate multiple
  7  # TE keys to different YAML files at once
  8  group: "GRPSERIAL"
  9  # A tag is a label to classify jobs when displaying results
 10  tag: ["a", "b"]
 11  # Build rules => how to build the benchmark
 12  build:
 13    # Files used to build this node
 14    # it can be a source file, a makefile...
 15    files: "@SRCPATH@/*.c"
 16    # ===== FOR ANY BUILD SYSTEM
 17    # args: arguments forwarded to the build system
 18    # envs: environment variables exported to the target shell.
 19    # Only one build system is expected to be used at once among the following:
 20
 21    autotools:
 22      # Use autogen instead of autoreconf
 23      autogen: true
 24      args: ['--disable-bootstrap']
 25      envs: []
 26    cmake:
 27      # sub build directory relative to PCVS tests build directory
 28      # It is mostly used when multiple configure have to be done using CMake
 29      sub_build_dir: 'my_personal_build_dir'
 30      args: ['-DCMAKE_BUILD_TYPE=Release']
 31      envs: []
 32    make:
 33      # default makefile target to invoke
 34      target: "all"
 35      # a list of macros can be used as placeholder to be replaced when
 36      # unrolling. A list is available at the end of this file.
 37      args: ['MPICC=@COMPILER_CC@']
 38      envs: []
 39      # Number of compilation jobs (-j option)
 40      # It can be the number explicitly or true to use the number defined in the machine profile
 41      jobs: true
 42    sources:
 43      # program binary (if needed)
 44      binary: "a.out"
 45      # extra cflags
 46      cflags: "extra cflags"
 47      # extra ldflags
 48      ldflags: "extra ldflags"
 49      envs: []
 50      # force explicit language if PCVS has trouble to autodetect
 51      lang: ['cc', 'cxx', 'fc']
 52    custom:
 53      # Really complex build scenario PCVS may not be able to handle
 54      program: "myscript.sh"
 55      args: ['']
 56      envs: ["VAR1=value", "VAR2=value"]
 57    validate:
 58      # same as validate node bellow but to validate the build process
 59      # instead of the run.
 60      [...]
 61    # dependency scheme. This can be:
 62    # a fully expanded test name
 63    # a TE node (like ".this_is_a_job" for the current node).
 64    # A circular detection is run.
 65    depends_on: ["this_is_another_test"]
 66    # directory where program should be built
 67    cwd: "dir/to/build"
 68    # variants required from profiles to run this job
 69    # IF these variants are not satisfied, jobs will be discarded.
 70    # it indicates not support for a specific variant.
 71    # There is no preset variants, all of them can be declared by the profile.
 72    # It can be found under compiler.$LANG.variants
 73    # The args and envs defined in the variants will be concatenated with the currents args and envs.
 74    variants:
 75      - openmp
 76      - accel
 77      - mpi
 78    # PM interaction, allow loading package/module BEFORE running jobs
 79    # This load will be done for every job, it may create overhead.
 80    # currently, only spack recipes & module will be allowed.
 81    # If a spack recipe is not available, it will be installed first.
 82    package_manager:
 83      spack:
 84        - protobuf@3.1.1
 85        - gcc@7.3.0
 86      module:
 87        - protobuf
 88        - gnu/gcc/7.3.0
 89
 90  run: &run_part
 91    # path to program (defaults to build.sources.binary if not set)
 92    program: "./a.out"
 93    # Scale the benchmark to build test scenarios.
 94    iterate:
 95      # subnodes are parsed relatively to the profile configuration.
 96      # There are expected to be declared to profiles, under "criterion"
 97      # only intersected values with profiles will be kept.
 98      n_mpi:
 99        values: [2, 4]
100      n_omp:
101        values: [1, 2]
102      # Special key to declare program-scope parameter matrix.
103      # One can use as many parameters to run the given program.
104      # criterion name will be given by the key
105      # criterion values will be given by the "values" field
106      # criterion label (used to generate test-name) will be given by "subtitle"
107      # parameter position will be given by "type": "argument" or "environment"
108      # numeric attributes allow using sequence to generate values (for more information see Series in configuration).
109      # for instance, to generate only power of 2 values:
110      #  => values: [{from: 0; to: 100, op: "mul": of: 2}]
111      program:
112        give_it_a_name:
113          numeric: true
114          # false send the argument to the wrapper of the program
115          # true (default) send the argument to the program itself
116          local: true
117          type: "argument"
118          values: ["-iter 1000", "-fast"]
119          subtitle: "i"
120    # directory where program should be built
121    cwd: "dir/to/build"
122    # dependency scheme, same rules as build.depends_on.
123    depends_on: ["this_is_another_run_test_in_the_same_file"]
124    # PM interaction, same as build.package_manager
125    package_manager:
126      spack:
127        - protobuf@3.1.1
128        - gcc@7.3.0
129      module:
130        - protobuf
131        - gnu/gcc/7.3.0
132  # How to validate jobs generated from this specification ?
133  # each of these subkeys are cumulative, all of them has to be true to
134  # mark the job as a success
135  validate:
136    # expected return code
137    expect_exit: 0
138    # time limit
139    time:
140      # max execution time the job should not exceed to succeed
141      mean: 10.0
142      # time measurement may vary. Some tolerance is allowed (only upper-bound)
143      tolerance: 2.0
144      # After this time, job will be automatically stopped and marked as failed
145      # note that PCVS cannot be able to handle every job kill mechanism.
146      # Our approach is to terminate the process which spawned the test on
147      # resources. With some batch-managers, it may lead to leaks.
148      hard_timeout: 40
149      # Mark the job as failed without killing it.
150      # This allow you to run the test for code correctness on a slow machine
151      # without testing for speed.
152      soft_timeout: 20
153    # Output formatting compliance
154    # It is based on rule definition as regex. These regexs has to be matched (or
155    # not) to pass.
156    match:
157      # rule name
158      label:
159        # the regex
160        expr: '^\d+(\.\d+) received$'
161        # Is the pattern allowed or forbidden in the output.
162        # True: The regex MUST match to success
163        # False: The regex MUST NOT match to success
164        # default: True
165        expect: true|false
166      label2: 'Total Elapsed: \d+\.\d+ sec.$'
167    # Analysis method
168    # based on dedicated Python module (builtin or user-made) to be run on every
169    # jobs to select the final status
170    # use-case: mark job status depending on its execution history over past
171    # runs.
172    analysis:
173      method: "<method>"
174    # Generic scenario, not handled by PCVS.
175    # Job info & output are provided to the script, returning 0 in case of
176    # success, non-zero otherwise.
177    script:
178      path: "/path/to/script"
179
180  # Post execution rules.
181  # artifacts can be stored to the final results to store job-specific data.
182  # Currently, artifacts only supports filesystem
183  artifact:
184    # relative to $BUILDPATH
185    obj1: "./path/1"
186    obj2: "./path/2"
187
188  # From the output, extract data to be stored as results.
189  # A regex is used to capture the metric from the standard output and is stored
190  # to the final result JSON file for later reference.
191  metrics:
192    # metric name
193    metric1:
194      # regex to match
195      key: "regex"
196    metric2:
197      key: "regex"
198      attributes:
199        # if multiple occurrences of the regex are found, only keep the first
200        # match. (default: false)
201        unique: true
202
203  # A list of properties to enable specific features.
204  # These attributes are common to build/run configurations
205  # if a different set of attributes are required for build & run a single TE,
206  # feel free to create TWO nodes, with a dependency on "build".
207  attributes:
208    # control if the current benchmark program should be prefixed by
209    # the runtime.program command (default: true)
210    # True: ./a.out -> mpirun ./a.out
211    # False: ./a.out -> ./a.out
212    command_wrap: true
213    # controls if the relative path defined by this TE should be converted to
214    # absolute path to avoid PATH resolution. (default: true)
215    # True: ./a.out -> @BUILDIR@/./a.out
216    # False: spack install -> spack install
217    path_resolution: true
218    # duplicates inputs defined by this TE for any generated test.
219    # For instance, if the current build directory provide a non-shareable
220    # resource, it may run into trouble when running concurrent tests from this
221    # specification. A dedicated directory is built for each subtest.
222    # (default: false)
223    copy_input: false
224    # duplicates outputs defined by this TE for any generated test.
225    # This is the opposite of copy_input, to allow generating concurrent artifacts
226    # from this single specification (default: false)
227    copy_output: false
228
229
230#########################################################
231
232# depicts an inheritance mechanism.
233# a key is defined as &key, as put on the "run" node above.
234# It can then be referred with "*key". Any subkeys are copied
235# to under the target node.
236# This can be used to factor keys among multiples job descriptions
237real_test:
238  build:
239    make:
240      target: all
241  run:
242    <<: *run_part

Configurations nodes

Compiler

 1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/compiler-scheme.yml
 2---
 3# List of programs to be used to compile
 4# a wide variety of programs
 5# Variants can add flavour to test requiring them.
 6# For instance a test asking for the variant 'openmp'
 7# will have its cflags extended with the content of
 8# openmp.args here. Variants override parent fields, except
 9# list objects, where more specialized content is appended
10# to the parent
11# C compiler
12compilers:
13  cc:
14    program: "/path/to/cc"
15    args: ["-Wall", "-foption"]
16    envs: ["MPI_CC=cc", "EXTRA_FLAGS=-foption"]
17    variants:
18      openmp:
19        args: ["-fopenmp"]
20      mpi:
21        program: "/path/to/mpicc"
22# The drawback here is to duplicate variants for each compiler
23# please use YAML anchors (&, *) when possible:
24# cc:
25#   variant: &variants_below
26#     openmp:
27#       args: ['-fopenmp']
28# cxx:
29#   variant: *variants_below
30
31# Manage the compiler through a package manager
32# Before any test to be run, the PM is invoked to load
33# everything required.
34# Note, only a single can be specified but both can be used
35# at the same time
36package_manager:
37  spack:
38    - "name@version+variants"
39  module:
40    - "group/flavor/version"

Criterion

 1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/criterion-scheme.yml
 2---
 3# Defines criterion values to be used during the validation
 4# Iterators are only valid if they have been declared
 5# previously by the runtime
 6# a criterion name
 7n_node:
 8  # the range of values it can take (any type)
 9  values: [1, 2, 3, 4]
10  # the prefix to use when labelling test name for this IT
11  subtitle: "N"
12n_proc:
13  values: [2, 4, 8]
14  subtitle: "p"
15n_mpi:
16  values: [2, 32]
17  subtitle: "n"
18n_omp:
19  values: [4, 8]
20  subtitle: "o"
21n_core:
22  values: [1, 2]
23  subtitle: "c"
24
25# When iterators are declared as 'numeric' by the runtime,
26# special syntaxes have been introduced to ease the definition
27# of complex series of number. These are called sequences and
28# can be used as a replacement of a single value. They map as
29# dict, where the toor name if one of:
30# - sequence: to create an additive sequence
31# - multiplication: to create a multiplicative sequence
32# - power: to create a 'power of' sequence
33# - factor: to create a selective sequence of values based on a factor
34# - powerof: to create a selective sequence of values based on a power
35#
36# Each operation comes with three parameters:
37# - from: lowerbound (inclusive)
38# - to: upperbound (inclusive)
39# - of: the 'stride/factor/power' to apply
40#
41# Examples:
42# - seq: {from: 2, to: 10, of: 2} --> [2, 4, 6, 8, 10]
43# - mul: {from: 2, to: 10, of: 2} --> [2, 4, 8]
44# - pow: {from: 2, to: 10, of: 2} --> [2, 4]
45# - factor: {from: 2, to: 10, of: 2} --> [2, 4, 6, 8, 10]
46# - powerof: {from: 2, to: 10, of: 2} --> [4, 9]
47# - powerof: {from: 1, to: 100, of 3} --> [1, 8, 27, 64]

Group

 1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/group-scheme.yml
 2---
 3# Please here defines nodes like regular TEs
 4# None of these templates will be unrolled
 5# first-level keys can then be used in `group' field
 6GRPMPI:
 7  run:
 8    iterate:
 9      inherit: ["n_mpi", "n_node", "n_core"]
10GRPMPI_NOTHREADS:
11  run:
12    iterate:
13      inherit: ["n_mpi", "n_node"]
14GRPOMP:
15  run:
16    iterate:
17      inherit: ["n_omp", "n_core"]
18GRPSERIAL:
19  run:
20    iterate:
21      inherit: []

Machine

 1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/machine-scheme.yml
 2---
 3name: 'localhost'
 4nodes: 1
 5# time limit to allocate resources properly
 6# Max number of allocation to be made concurrently
 7concurrent_run: 5
 8# Command to use when interacting with batch manager
 9job_manager:
10  mintime: 1000
11  maxtime: 1500
12  allocate:
13    program: "echo"
14    args: ""
15    wrapper: ""
16  remote:
17    program: "echo"
18    args: ""
19    wrapper: ""
20    #  batch:
21    # command: "srun"
22    # args: "-p sandy"
23    # wrapper: "/pat/to/batch_wrapper"
24# default partition to load (None by default)
25default_partition: 'knl'
26# defines partitions overring settings above
27partitions:
28  - name: 'knl'
29    nodes: 4
30    cores_per_node: 272
31    job_manager:
32      mintime: 10
33  - name: 'arm'
34    nodes: 100
35    cores_per_node: 32
36    job_manager:
37      allocate:
38        program: 'echo'
39        args: ""
40        wrapper: ""

Runtime

 1# yaml-language-server: $schema=https://raw.githubusercontent.com/cea-hpc/pcvs/refs/heads/master/pcvs/schemes/generated/runtime-scheme.yml
 2---
 3# program path to run any program (can be a name if in $PATH)
 4program: "/path/to/command"
 5# arguments to use with the program above
 6args: "--verbose"
 7
 8# Declaration of criterion
 9# Here is set any element to describe what an criterion iterator does
10# A criterion is small component a test is relying on to be validated
11# For instance, an MPI program has the following components:
12# - number of nodes to be executed on
13# - number of MPI ranks
14# - number of cores allocated for each MPI rank
15# - Type of network to enable inter process communication
16# A single test is a combination of one single value picked up from
17# these components.
18criterions:
19    # please use a short, without-space names
20    iterator_name:
21        # option known by the runtime to handle a value of this component
22        option: ""
23        # Is the sequence of values only number ?
24        numeric: true
25        # two cases:
26        # - argument -> $PROGRAM $OPTION$VALUE ./a.out
27        # - environment -> $OPTION=$VALUE $PROGRAM ./a.out
28        type: "environment"
29        # should the value put before or after the option name ?
30        position: "before"
31        # value aliasing
32        # To make values uniform across any tests, a configuration may rename
33        # the value to the proper label for its runtime
34        # for instance, 'ib' is used by iterators, translating to 'openib'
35        # when using OpenMPI
36        aliases:
37            ib: "openib"
38            tcp: "tcp"
39            shmem: "sm"
40            ptl: "portals"
41
42# Manage the runtime through a package manager
43# Before any test to be run, the PM is invoked to load
44# everything required.
45# Note, only a single can be specified but both can be used
46# at the same time
47package_manager:
48    # for Spack, the spec must refer to a single concretization
49    spack:
50        - "name@version+variants"
51    module:
52        - "group/flavor/version"