# Test reporting bodies
# TAP specification at
# http://testanything.org/tap-specification.html
@if minimum_version(3.9)
bundle agent testing_ok_if(classname, message, error, trace, format)
# @brief Report outcome for test on `classname` in format `format` with optional `error` and `trace`
# @param classname The class name
# @param message The test message
# @param error The error to report, if the class is not defined
# @param trace The error trace detail to report, if the class is not defined
# @param format TAP (immediate output) or jUnit or anything else for delayed TAP
#
# **See also:** `testing_junit_report`, `testing_tap_report`, `testing_ok`, `testing_todo`, and `testing_skip`
{
  vars:
    "next_testing_$(classname)"
      int => length(
        classesmatching("testing_.*", "testing_(passed|failed|skipped|todo)")
      );

    "next_testing_$(classname)_failed"
      int => length(
        classesmatching("testing_.*", "testing_(passed|failed|skipped|todo)")
      );

  classes:
    "tap" expression => strcmp('tap', string_downcase($(format)));

    "testing_$(classname)"
      expression => $(classname),
      scope => "namespace",
      meta => {
        "testing_passed",
        "error=$(error)",
        "trace=$(trace)",
        "message=$(message)",
      };

    "testing_$(classname)_failed"
      not => $(classname),
      scope => "namespace",
      meta => {
        "testing_failed",
        "error=$(error)",
        "trace=$(trace)",
        "message=$(message)",
      };

  reports:
    inform_mode::
      "$(this.bundle): adding testing report for class $(classname) at position $(next_testing_$(classname))";

    "tap.testing_$(classname)"::
      "$(const.n)ok $(message)";

    "tap.testing_$(classname)_failed"::
      "$(const.n)not ok $(message)";
}

bundle agent testing_ok(classname, message, format)
# @brief Report expected success in format `format` for `classname` and its test
# @param classname The class name
# @param message The test message
# @param format TAP (immediate output) or jUnit or anything else for delayed TAP
#
# This bundle calls `testing_ok_if` expecting `classname` to be defined and thus the
# test to be a success; the error and trace reflect that.
#
# **See also:** `testing_junit_report`, `testing_tap_report`, `testing_ok_if`, `testing_todo`, and `testing_skip`
{
  methods:
    ""
      usebundle => testing_ok_if(
        $(classname),
        $(message),
        "unexpected error for $(classname)",
        "no error trace available",
        $(format)
      );
}

bundle agent testing_skip(classname, message, format)
# @brief Report skipped `classname` in format `format`
# @param classname The class name
# @param message The test message
# @param format TAP (immediate output) or jUnit or anything else for delayed TAP
#
# This bundle reports that `classname` was skipped regardless of whether it's
# defined.
#
# **See also:** `testing_junit_report`, `testing_tap_report`, `testing_ok_if`, `testing_todo`, and `testing_ok`
{
  vars:
    "next_testing_$(classname)"
      int => length(
        classesmatching("testing_.*", "testing_(passed|failed|skipped|todo)")
      );

  classes:
    "tap" expression => strcmp('tap', string_downcase($(format)));

    "testing_$(classname)"
      scope => "namespace",
      meta => { "testing_skipped", "message=$(message)" };

  reports:
    inform_mode::
      "$(this.bundle): adding testing skip report for class $(classname) at position $(next_testing_$(classname))";

    tap::
      "$(const.n)ok # SKIP $(message)";
}

bundle agent testing_todo(classname, message, format)
# @brief Report TODO `classname` in format `format`
# @param classname The class name
# @param message The test message
# @param format TAP (immediate output) or jUnit or anything else for delayed TAP
#
# This bundle reports that `classname` was skipped regardless of whether it's
# defined.
#
# **See also:** `testing_junit_report`, `testing_tap_report`, `testing_ok_if`, `testing_skip`, and `testing_ok`
{
  vars:
    "next_testing_$(classname)"
      int => length(
        classesmatching("testing_.*", "testing_(passed|failed|skipped|todo)")
      );

  classes:
    "tap" expression => strcmp('tap', string_downcase($(format)));

    "testing_$(classname)"
      scope => "namespace",
      meta => { "testing_todo", "message=$(message)" };

  reports:
    inform_mode::
      "$(this.bundle): adding testing TODO report for class $(classname) at position $(next_testing_$(classname))";

    tap::
      "$(const.n)ok # TODO $(message)";
}

bundle agent testing_tap_report(outfile)
# @brief Report all test messages in TAP format to `outfile`
# @param outfile A text file with the final TAP report or empty `` for STDOUT report
#
# Note that the TAP format ignores error messages, trace messages, and class names.
#
# **See also:** `testing_junit_report`, `testing_tap_report`, `testing_ok_if`, `testing_todo`, `testing_skip`, `testing_tap_bailout`, and `testing_ok`
{
  methods:
    "" usebundle => testing_generic_report('TAP', $(outfile));
}

bundle agent testing_junit_report(outfile)
# @brief Report all test knowledge in jUnit format to `outfile`
# @param outfile A XML file with the final jUnit report or empty `` for STDOUT report
#
# **See also:** `testing_tap_report`, `testing_ok_if`, `testing_todo`, `testing_skip`, and `testing_ok`
{
  methods:
    "" usebundle => testing_generic_report('jUnit', $(outfile));
}

bundle agent testing_generic_report(format, outfile)
# @brief Report all test knowledge in jUnit format to `outfile`
# @param format The output format, either `jUnit` or `TAP` (case is ignored)
# @param outfile A file with the final report or empty for STDOUT
#
# Note that jUnit output to STDOUT will most likely be truncated due to the 4K
# limitation on string lengths.
#
# **See also:** `testing_tap_report`, `testing_ok_if`, `testing_todo`, `testing_skip`, and `testing_ok`
{
  classes:
    "junit" expression => strcmp('junit', string_downcase($(format)));
    "tap" expression => strcmp('tap', string_downcase($(format)));
    "stdout" expression => strcmp('', $(outfile));
    "tofile" not => strcmp('', $(outfile));

  vars:
    "failed" slist => classesmatching("testing_.*", testing_failed);
    "passed" slist => classesmatching("testing_.*", testing_passed);
    "skipped" slist => classesmatching("testing_.*", testing_skipped);
    "todo" slist => classesmatching("testing_.*", testing_todo);
    "count_passed" int => length(passed);
    "count_skipped" int => length(skipped);
    "count_todo" int => length(todo);
    "count_failed" int => length(failed);

    "count_total"
      string => format(
        "%d",
        sum(
          '[$(count_failed), $(count_passed), $(count_skipped), $(count_todo)]'
        )
      );

    "timestamp" string => strftime("localtime", "%FT%T", now());
    "tests_passed" data => '[]';

    "tests_passed"
      data => mergedata(
        tests_passed,
        format(
          '[{ "testcase": "%s", "test_offset": %d, "test_message": "%s" }]',
          regex_replace($(passed), "^testing_", "", "T"),
          "$(testing_ok_if.next_$(passed))",
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(passed)))),
            "^message=",
            "",
            "T"
          ),
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(passed)))),
            "^message=",
            "",
            "T"
          )
        )
      );

    "tests_failed" data => '[]';

    "tests_failed"
      data => mergedata(
        tests_failed,
        format(
          '[{ "testcase": "%s", "failure": true, "fail_message": "%s", "trace_message": "%s", "test_offset": %d, "test_message": "%s" }]',
          regex_replace($(failed), "^testing_", "", "T"),
          regex_replace(
            join(",", grep("^error=.*", getclassmetatags($(failed)))),
            "^error=",
            "",
            "T"
          ),
          regex_replace(
            join(",", grep("^trace=.*", getclassmetatags($(failed)))),
            "^trace=",
            "",
            "T"
          ),
          "$(testing_ok_if.next_$(failed))",
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(failed)))),
            "^message=",
            "",
            "T"
          ),
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(failed)))),
            "^message=",
            "",
            "T"
          )
        )
      );

    "tests_skipped" data => '[]';

    "tests_skipped"
      data => mergedata(
        tests_skipped,
        format(
          '[{ "testcase": "%s", "test_offset": %d, "test_message": "%s", "skip": true }]',
          regex_replace($(skipped), "^testing_", "", "T"),
          "$(testing_skip.next_$(skipped))",
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(skipped)))),
            "^message=",
            "",
            "T"
          ),
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(skipped)))),
            "^message=",
            "",
            "T"
          )
        )
      );

    "tests_todo" data => '[]';

    "tests_todo"
      data => mergedata(
        tests_todo,
        format(
          '[{ "testcase": "%s", "test_offset": %d, "test_message": "%s", "todo": true }]',
          regex_replace($(todo), "^testing_", "", "T"),
          "$(testing_todo.next_$(todo))",
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(todo)))),
            "^message=",
            "",
            "T"
          ),
          regex_replace(
            join(",", grep("^message=.*", getclassmetatags($(todo)))),
            "^message=",
            "",
            "T"
          )
        )
      );

    inform_mode::
      "out"
        string => format(
          "counts = %d/%d/%d/%d/%d failed = %S, passed = %S, skipped = %S, todo = %S, failed = %S; tests = %S+%S+%S+%S",
          $(count_total),
          $(count_passed),
          $(count_skipped),
          $(count_todo),
          $(count_failed),
          failed,
          passed,
          skipped,
          todo,
          failed,
          tests_passed,
          tests_skipped,
          tests_todo,
          tests_failed
        );

    junit.stdout::
      "junit_out"
        string => string_mustache(
          readfile("$(this.promise_dirname)/templates/junit.mustache", 4k),
          bundlestate("testing_generic_report")
        );

    tap::
      "tap_tests"
        data => mergedata(
          tests_passed, tests_failed, tests_skipped, tests_todo
        );

      "tap_json"
        string => string_mustache(
          concat(
            '[ ',
            '{{#-top-}}',
            '"',
            '{{#failure}}not ok {{/failure}}',
            '{{^failure}}ok {{/failure}}',
            '{{test_offset}} {{test_message}}',
            '{{#skip}} # SKIP {{/skip}}',
            '{{#todo}} # TODO {{/todo}}',
            '", ',
            '{{/-top-}}',
            ' ]'
          ),
          tap_tests
        );

      "tap_results" data => parsejson("${tap_json}");

    tap.inform_mode::
      "tap_tests_info" string => format("%S", tap_tests);
      "tap_results_info" string => format("%S", tap_results);

  files:
    junit.tofile::
      "$(outfile)"
        create => "true",
        template_data => bundlestate("testing_generic_report"),
        template_method => "mustache",
        edit_template => "$(this.promise_dirname)/templates/junit.mustache";

    tap.tofile::
      "$(outfile)"
        create => "true",
        template_data => bundlestate("testing_generic_report"),
        template_method => "mustache",
        edit_template => "$(this.promise_dirname)/templates/tap.mustache";

  reports:
    junit.stdout::
      "$(const.n)$(junit_out)";

    tap.stdout::
      "$(const.n)1..$(count_total)";

    inform_mode::
      "$(this.bundle): report summary: $(out)";

    tap.inform_mode::
      "$(this.bundle): TAP report summary: $(tap_tests_info)";
      "$(this.bundle): TAP report results summary: $(tap_results_info)";
}

bundle agent testing_tap_bailout(reason)
# @brief Bail out in TAP format **immediately**
# @param reason the bailout reason
#
# **See also:** `testing_tap_report`, `testing_ok_if`, `testing_todo`, `testing_skip`, and `testing_ok`
{
  reports:
    "$(const.n)Bail out! $(reason)";
}

bundle agent testing_usage_example
# @brief Simple demo of testing_junit_report and testing_tap_report testing.cf usage
#
# You can run it like this: `cf-agent -K ./testing.cf -b testing_usage_example`
# Or for extra debugging, you can run it like this: `cf-agent -KI ./testing.cf -b testing_usage_example`
#
# You can either use `tap` as the `format` parameter for any testing bundle, in
# which case you get immediate TAP output, OR you can use anything else, in
# which case you can still get TAP output but at the end.
#
# So your use cases are:
#
# * format=jUnit, then testing_junit_report(''): all jUnit to STDOUT, output at end
# * format=TAP, then testing_tap_report(''): all TAP to STDOUT, immediate output
# * format=delayed_TAP, then testing_tap_report(MYFILE): all TAP to MYFILE, output at end
# * format=jUnit, then testing_jUnit_report(MYFILE): all jUnit to MYFILE, output at end
{
  classes:
    "reported_class" scope => "namespace";

  methods:
    "" usebundle => testing_ok("reported_class", "ok message", "TAP");

    ""
      usebundle => testing_ok_if(
        "missing_class", "missing message", "error1", "error trace 1", "TAP"
      );

    ""
      usebundle => testing_ok_if(
        "missing_class2", "missing message2", "error2", "error trace 2", "TAP"
      );

    "" usebundle => testing_skip("skipped_class", "we skipped this", "TAP");
    "" usebundle => testing_todo("todo_class", "we need to do this", "TAP");

    # output the reports to some files
    # "" usebundle => testing_junit_report("/var/cfengine/outputs/junit.xml");
    # "" usebundle => testing_tap_report("/var/cfengine/outputs/tap.txt");
    # output the reports to STDOUT
    "" usebundle => testing_junit_report('');
    "" usebundle => testing_tap_report('');
}
@endif
