Added cdash framework for pjsip tests. Currently there is only test framework for GNU
git-svn-id: https://svn.pjsip.org/repos/pjproject/trunk@2399 74dad513-b988-da41-8d7b-12977e46ad98
diff --git a/tests/cdash/README.TXT b/tests/cdash/README.TXT
new file mode 100644
index 0000000..0d09965
--- /dev/null
+++ b/tests/cdash/README.TXT
@@ -0,0 +1,50 @@
+
+ PJSIP CDASH AUTOMATED TESTS
+ --------------------------------
+
+
+1. What is this
+
+ This directory contains the scripts to run the automated, Python based tests
+of PJSIP source codes, across platforms, and submit the test results to a CDash
+test monitoring dashboard.
+
+ Stuffs that are included in the test scope:
+ - configure (for GNU platforms, e.g. Linux, msys, and MacOS X)
+ - build
+ - standard unit tests (pjlib-test, pjsip-test, etc.)
+ - pjsua's Python based blackbox tests
+
+
+2. Requirements
+
+ To run this test, you will need:
+ - Python (2.4 or later)
+ - curl (http://curl.haxx.se)
+ - a CDash server to receive test submissions (http://www.cdash.org)
+ - ccdash to submit the tests (http://trac.pjsip.org/ccdash)
+
+
+3. Configuration
+
+ Create a Python configuration file by copying from "cfg_site_sample.py".
+Save it as "cfg_site.py". You may create more than one configurations for your
+site and save them as different files.
+
+
+4. Running
+
+ To execute tests for GNU based targets:
+
+ $ python main.py cfg_gnu
+
+
+ To execute tests for MSVC based target:
+
+ $ python main.py cfg_msvc
+
+
+ If you have a different site configuration file, you may specify it in the
+arguments, e.g.:
+
+ $ python main.py cfg_gnu my_site_config
diff --git a/tests/cdash/builder.py b/tests/cdash/builder.py
new file mode 100644
index 0000000..6336a3d
--- /dev/null
+++ b/tests/cdash/builder.py
@@ -0,0 +1,262 @@
+import ccdash
+import os
+import platform
+import re
+import subprocess
+import sys
+import time
+
+class Operation:
+ """\
+ The Operation class describes the individual ccdash operation to be
+ performed.
+
+ """
+ # Types:
+ UPDATE = "update" # Update operation
+ CONFIGURE = "configure" # Configure operation
+ BUILD = "build" # Build operation
+ TEST = "test" # Unit test operation
+
+ def __init__(self, type, cmdline, name="", wdir=""):
+ self.type = type
+ self.cmdline = cmdline
+ self.name = name
+ self.wdir = wdir
+ if self.type==self.TEST and not self.name:
+ raise "name required for tests"
+
+ def encode(self, base_dir):
+ s = [self.type]
+ if self.type == self.TEST:
+ s.append(self.name)
+ if self.type != self.UPDATE:
+ s.append(self.cmdline)
+ s.append("-w")
+ if self.wdir:
+ s.append(base_dir + "/" + self.wdir)
+ else:
+ s.append(base_dir)
+ return s
+
+
+#
+# Update operation
+#
+update_ops = [Operation(Operation.UPDATE, "")]
+
+#
+# The standard library tests (e.g. pjlib-test, pjsip-test, etc.)
+#
+std_test_ops= [
+ Operation(Operation.TEST, "./pjlib-test-$SUFFIX", name="pjlib test",
+ wdir="pjlib/bin"),
+ Operation(Operation.TEST, "./pjlib-util-test-$SUFFIX",
+ name="pjlib-util test", wdir="pjlib-util/bin"),
+ Operation(Operation.TEST, "./pjnath-test-$SUFFIX", name="pjnath test",
+ wdir="pjnath/bin"),
+ Operation(Operation.TEST, "./pjmedia-test-$SUFFIX", name="pjmedia test",
+ wdir="pjmedia/bin"),
+ Operation(Operation.TEST, "./pjsip-test-$SUFFIX", name="pjsip test",
+ wdir="pjsip/bin")
+]
+
+#
+# These are operations to build the software on GNU/Posix systems
+#
+gnu_build_ops = [
+ Operation(Operation.CONFIGURE, "./configure"),
+ Operation(Operation.BUILD, "make distclean; make dep && make; cd pjsip-apps/src/python; python setup.py clean build"),
+ #Operation(Operation.BUILD, "python setup.py clean build",
+ # wdir="pjsip-apps/src/python")
+]
+
+#
+# These are pjsua Python based unit test operations
+#
+def build_pjsua_test_ops():
+ ops = []
+ cwd = os.getcwd()
+ os.chdir("../pjsua")
+ os.system("python runall.py --list > list")
+ f = open("list", "r")
+ for e in f:
+ e = e.rstrip("\r\n ")
+ (mod,param) = e.split(None,2)
+ name = mod[4:mod.find(".py")] + "_" + \
+ param[param.find("/")+1:param.find(".py")]
+ ops.append(Operation(Operation.TEST, "python run.py " + e, name=name,
+ wdir="tests/pjsua"))
+ os.chdir(cwd)
+ return ops
+
+#
+# Get gcc version
+#
+def gcc_version(gcc):
+ proc = subprocess.Popen(gcc + " -v", stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, shell=True)
+ ver = ""
+ while True:
+ s = proc.stdout.readline()
+ if not s:
+ break
+ if s.find("gcc version") >= 0:
+ ver = s.split(None, 3)[2]
+ break
+ proc.wait()
+ return "gcc-" + ver
+
+#
+# Test config
+#
+class BaseConfig:
+ def __init__(self, base_dir, url, site, group, options=None):
+ self.base_dir = base_dir
+ self.url = url
+ self.site = site
+ self.group = group
+ self.options = options
+
+#
+# Base class for test configurator
+#
+class TestBuilder:
+ def __init__(self, config, build_config_name="",
+ user_mak="", config_site="", exclude=[], not_exclude=[]):
+ self.config = config # BaseConfig instance
+ self.build_config_name = build_config_name # Optional build suffix
+ self.user_mak = user_mak # To be put in user.mak
+ self.config_site = config_site # To be put in config_s..
+ self.saved_user_mak = "" # To restore user.mak
+ self.saved_config_site = "" # To restore config_s..
+ self.exclude = exclude # List of exclude pattern
+ self.not_exclude = not_exclude # List of include pattern
+ self.ccdash_args = [] # ccdash cmd line
+
+ def stamp(self):
+ return time.strftime("%Y%m%d-%H%M", time.localtime())
+
+ def pre_action(self):
+ # Override user.mak
+ name = self.config.base_dir + "/user.mak"
+ if os.access(name, os.F_OK):
+ f = open(name, "r")
+ self.saved_user_mak = f.read()
+ f.close()
+ if True:
+ f = open(name, "wt")
+ f.write(self.user_mak)
+ f.close()
+ # Override config_site.h
+ name = self.config.base_dir + "/pjlib/include/pj/config_site.h"
+ if os.access(name, os.F_OK):
+ f = open(name, "r")
+ self.saved_config_site= f.read()
+ f.close()
+ if True:
+ f = open(name, "wt")
+ f.write(self.config_site)
+ f.close()
+
+
+ def post_action(self):
+ # Restore user.mak
+ name = self.config.base_dir + "/user.mak"
+ if self.saved_user_mak:
+ f = open(name, "wt")
+ f.write(self.saved_user_mak)
+ f.close()
+ else:
+ os.remove(name)
+ # Restore config_site.h
+ name = self.config.base_dir + "/pjlib/include/pj/config_site.h"
+ if self.saved_config_site:
+ f = open(name, "wt")
+ f.write(self.saved_config_site)
+ f.close()
+ else:
+ os.remove(name)
+
+ def build_tests(self):
+ # This should be overridden by subclasses
+ pass
+
+ def execute(self):
+ if len(self.ccdash_args)==0:
+ self.build_tests()
+ self.pre_action()
+ counter = 0
+ for a in self.ccdash_args:
+ # Check if this test is in exclusion list
+ fullcmd = " ".join(a)
+ excluded = False
+ included = False
+ for pat in self.exclude:
+ if re.search(pat, fullcmd) != None:
+ excluded = True
+ break
+ if excluded:
+ for pat in self.not_exclude:
+ if re.search(pat, fullcmd) != None:
+ included = True
+ break
+ if excluded and not included:
+ print "Skipping test '%s'.." % (fullcmd)
+ continue
+
+ #a.extend(["-o", "/tmp/xx" + a[0] + ".xml"])
+ #print a
+ #a = ["ccdash.py"].extend(a)
+ b = ["ccdash.py"]
+ b.extend(a)
+ a = b
+ #print a
+ ccdash.main(a)
+ counter = counter + 1
+ self.post_action()
+
+
+#
+# GNU test configurator
+#
+class GNUTestBuilder(TestBuilder):
+ def __init__(self, config, build_config_name="", user_mak="", \
+ config_site="", cross_compile="", exclude=[], not_exclude=[]):
+ TestBuilder.__init__(self, config, build_config_name=build_config_name,
+ user_mak=user_mak, config_site=config_site,
+ exclude=exclude, not_exclude=not_exclude)
+ self.cross_compile = cross_compile
+ if self.cross_compile and self.cross_compile[-1] != '-':
+ self.cross_compile.append("-")
+
+ def build_tests(self):
+ if self.cross_compile:
+ suffix = self.cross_compile
+ build_name = suffix + gcc_version(self.cross_compile + "gcc")
+ else:
+ proc = subprocess.Popen(self.config.base_dir+"/config.guess",
+ stdout=subprocess.PIPE)
+ suffix = proc.stdout.readline().rstrip(" \r\n")
+ build_name = suffix+"-"+gcc_version(self.cross_compile + "gcc")
+
+ if self.build_config_name:
+ build_name = build_name + "-" + self.build_config_name
+ cmds = []
+ cmds.extend(update_ops)
+ cmds.extend(gnu_build_ops)
+ cmds.extend(std_test_ops)
+ cmds.extend(build_pjsua_test_ops())
+ self.ccdash_args = []
+ for c in cmds:
+ c.cmdline = c.cmdline.replace("$SUFFIX", suffix)
+ args = c.encode(self.config.base_dir)
+ args.extend(["-U", self.config.url,
+ "-S", self.config.site,
+ "-T", self.stamp(),
+ "-B", build_name,
+ "-G", self.config.group])
+ args.extend(self.config.options)
+ self.ccdash_args.append(args)
+
+
diff --git a/tests/cdash/cfg_gnu.py b/tests/cdash/cfg_gnu.py
new file mode 100644
index 0000000..55bb82b
--- /dev/null
+++ b/tests/cdash/cfg_gnu.py
@@ -0,0 +1,34 @@
+import builder
+import os
+import sys
+
+# Each configurator must export this function
+def create_builder(args):
+ # (optional) args format:
+ # site configuration module. If not specified, "cfg_site" is implied
+
+ if len(args)>0:
+ file = args[0]
+ else:
+ file = "cfg_site"
+
+ if os.access(file+".py", os.F_OK) == False:
+ print "Error: file '%s.py' doesn't exist." % (file)
+ sys.exit(1)
+
+ cfg_site = __import__(file)
+ test_cfg = builder.BaseConfig(cfg_site.BASE_DIR, \
+ cfg_site.URL, \
+ cfg_site.SITE_NAME, \
+ cfg_site.GROUP, \
+ cfg_site.OPTIONS)
+
+ builders = [
+ builder.GNUTestBuilder(test_cfg, build_config_name="default",
+ user_mak="export CFLAGS+=-Wall\n",
+ config_site="#define PJ_TODO(x)\n",
+ exclude=cfg_site.EXCLUDE,
+ not_exclude=cfg_site.NOT_EXCLUDE)
+ ]
+
+ return builders
diff --git a/tests/cdash/cfg_site_sample.py b/tests/cdash/cfg_site_sample.py
new file mode 100644
index 0000000..147aa49
--- /dev/null
+++ b/tests/cdash/cfg_site_sample.py
@@ -0,0 +1,25 @@
+import builder
+
+# Your site name
+SITE_NAME="Newham2"
+
+# The URL where tests will be submitted to
+URL = "http://192.168.0.2/dash/submit.php?project=PJSIP"
+
+# Test group
+GROUP = "Experimental"
+
+# PJSIP base directory
+BASE_DIR = "/root/project/pjproject"
+
+# List of additional ccdash options
+#OPTIONS = ["-o", "out.xml", "-y"]
+OPTIONS = ["-o", "out.xml"]
+
+# List of regular expression of test patterns to be excluded
+EXCLUDE = [".*"]
+
+# List of regular expression of test patterns to be included (even
+# if they match EXCLUDE patterns)
+NOT_EXCLUDE = ["run.py mod_run.*100_simple"]
+#"configure", "update", "build.*make", "build", "run.py mod_run.*100_simple"]
diff --git a/tests/cdash/inc_test.py b/tests/cdash/inc_test.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/tests/cdash/inc_test.py
@@ -0,0 +1 @@
+
diff --git a/tests/cdash/main.py b/tests/cdash/main.py
new file mode 100644
index 0000000..9f707bd
--- /dev/null
+++ b/tests/cdash/main.py
@@ -0,0 +1,20 @@
+import sys
+
+if len(sys.argv)==1:
+ print "Usage: main.py cfg_file [cfg_site]"
+ print "Example:"
+ print " main.py cfg_gnu"
+ print " main.py cfg_gnu custom_cfg_site"
+ sys.exit(1)
+
+
+args = []
+args.extend(sys.argv)
+args.remove(args[1])
+args.remove(args[0])
+
+cfg_file = __import__(sys.argv[1])
+builders = cfg_file.create_builder(args)
+
+for builder in builders:
+ builder.execute()