jenkins/make_test_output.py

"""Gathers output from test runs and create an XML file in JUnit format.

The output files from the individual tests have been written in a directory
structure like:

  $DIR/joblog  (output from "parallel --joblog joblog")
  $DIR/logs/1/cpp/stdout
  $DIR/logs/1/cpp/stderr
  $DIR/logs/1/csharp/stdout
  $DIR/logs/1/csharp/stderr
  $DIR/logs/1/java_jdk7/stdout
  $DIR/logs/1/java_jdk7/stderr
  etc.

This script bundles them into a single output XML file so Jenkins can show
detailed test results.  It runs as the last step before the Jenkins build
finishes.
"""

import os;
import sys;
from yattag import Doc
from collections import defaultdict

def readtests(basedir):
  tests = defaultdict(dict)

  # Sample input (note: separators are tabs).
  #
  # Seq	Host	Starttime	Runtime	Send	Receive	Exitval	Signal	Command
  # 1	:	1456263838.313	0.005	0	0	0	0	echo A
  with open(basedir + "/joblog") as jobs:
    firstline = next(jobs)
    for line in jobs:
      values = line.split("\t")

      name = values[8].split()[-1]
      test = tests[name]
      test["name"] = name
      test["time"] = values[3]

      exitval = values[6]
      if int(exitval):
        # We don't have a more specific message.  User should look at stderr.
        test["failure"] = "TEST FAILURE"
      else:
        test["failure"] = False

  for testname in os.listdir(basedir + "/logs/1"):
    test = tests[testname]

    with open(basedir + "/logs/1/" + testname + "/stdout") as f:
      test["stdout"] = f.read()

    with open(basedir + "/logs/1/" + testname + "/stderr") as f:
      test["stderr"] = f.read()

  # The cpp test is special since it doesn't run under parallel so doesn't show
  # up in the job log.
  tests["cpp"]["name"] = "cpp"

  with open(basedir + '/logs/1/cpp/build_time', 'r') as f:
    tests["cpp"]["time"] = f.read().strip()
  tests["cpp"]["failure"] = False

  ret = tests.values()
  ret.sort(key=lambda x: x["name"])

  return ret

def genxml(tests):
  doc, tag, text = Doc().tagtext()

  with tag("testsuites"):
    with tag("testsuite", name="Protobuf Tests"):
      for test in tests:
        with tag("testcase", name=test["name"], classname=test["name"],
                             time=test["time"]):
          with tag("system-out"):
            text(test["stdout"])
          with tag("system-err"):
            text(test["stderr"])
          if test["failure"]:
            with tag("failure"):
              text(test["failure"])

  return doc.getvalue()

sys.stderr.write("make_test_output.py: writing XML from directory: " +
                 sys.argv[1] + "\n");
print genxml(readtests(sys.argv[1]))