Alexandre Lision | 8af73cb | 2013-12-10 14:11:20 -0500 | [diff] [blame] | 1 | # $Id$ |
| 2 | import os |
| 3 | import sys |
| 4 | import time |
| 5 | import re |
| 6 | import shutil |
| 7 | |
| 8 | PYTHON = os.path.basename(sys.executable) |
| 9 | |
| 10 | # Usage: |
| 11 | # runall.py [test-to-resume] |
| 12 | |
| 13 | |
| 14 | # Initialize test list |
| 15 | tests = [] |
| 16 | |
| 17 | # Excluded tests (because they fail?) |
| 18 | excluded_tests = [ "svn", |
| 19 | "pyc", |
| 20 | "scripts-call/150_srtp_2_1", # SRTP optional 'cannot' call SRTP mandatory |
| 21 | "scripts-call/150_srtp_2_3.py", # temporarily disabled until #1267 done |
| 22 | "scripts-call/301_ice_public_a.py", # Unreliable, proxy returns 408 sometimes |
| 23 | "scripts-call/301_ice_public_b.py", # Doesn't work because OpenSER modifies SDP |
| 24 | "scripts-pres/200_publish.py", # Ok from cmdline, error from runall.py |
| 25 | "scripts-media-playrec/100_resample_lf_8_11.py", # related to clock-rate 11 kHz problem |
| 26 | "scripts-media-playrec/100_resample_lf_8_22.py", # related to clock-rate 22 kHz problem |
| 27 | "scripts-media-playrec/100_resample_lf_11" # related to clock-rate 11 kHz problem |
| 28 | ] |
| 29 | |
| 30 | # Add basic tests |
| 31 | for f in os.listdir("scripts-run"): |
| 32 | tests.append("mod_run.py scripts-run/" + f) |
| 33 | |
| 34 | # Add basic call tests |
| 35 | for f in os.listdir("scripts-call"): |
| 36 | tests.append("mod_call.py scripts-call/" + f) |
| 37 | |
| 38 | # Add presence tests |
| 39 | for f in os.listdir("scripts-pres"): |
| 40 | tests.append("mod_pres.py scripts-pres/" + f) |
| 41 | |
| 42 | # Add mod_sendto tests |
| 43 | for f in os.listdir("scripts-sendto"): |
| 44 | tests.append("mod_sendto.py scripts-sendto/" + f) |
| 45 | |
| 46 | # Add mod_media_playrec tests |
| 47 | for f in os.listdir("scripts-media-playrec"): |
| 48 | tests.append("mod_media_playrec.py scripts-media-playrec/" + f) |
| 49 | |
| 50 | # Add mod_pesq tests |
| 51 | for f in os.listdir("scripts-pesq"): |
| 52 | tests.append("mod_pesq.py scripts-pesq/" + f) |
| 53 | |
| 54 | # Add recvfrom tests |
| 55 | for f in os.listdir("scripts-recvfrom"): |
| 56 | tests.append("mod_recvfrom.py scripts-recvfrom/" + f) |
| 57 | |
| 58 | # Add sipp tests |
| 59 | for f in os.listdir("scripts-sipp"): |
| 60 | if f.endswith(".xml"): |
| 61 | tests.append("mod_sipp.py scripts-sipp/" + f) |
| 62 | |
| 63 | # Filter-out excluded tests |
| 64 | for pat in excluded_tests: |
| 65 | tests = [t for t in tests if t.find(pat)==-1] |
| 66 | |
| 67 | |
| 68 | resume_script="" |
| 69 | shell_cmd="" |
| 70 | |
| 71 | # Parse arguments |
| 72 | sys.argv.pop(0) |
| 73 | while len(sys.argv): |
| 74 | if sys.argv[0]=='/h' or sys.argv[0]=='-h' or sys.argv[0]=='--help' or sys.argv[0]=='/help': |
| 75 | sys.argv.pop(0) |
| 76 | print "Usage:" |
| 77 | print " runall.py [OPTIONS] [run.py-OPTIONS]" |
| 78 | print "OPTIONS:" |
| 79 | print " --list" |
| 80 | print " List the tests" |
| 81 | print " --list-xml" |
| 82 | print " List the tests as XML format suitable for ccdash" |
| 83 | print " --resume,-r RESUME" |
| 84 | print " RESUME is string/substring to specify where to resume tests." |
| 85 | print " If this argument is omited, tests will start from the beginning." |
| 86 | print " --shell,-s SHELL" |
| 87 | print " Run the tests with the specified SHELL cmd. This can also be" |
| 88 | print " used to run the test with ccdash. Example:" |
| 89 | print " --shell '/bin/sh -c'" |
| 90 | print " run.py-OPTIONS are applicable here" |
| 91 | sys.exit(0) |
| 92 | elif sys.argv[0] == '-r' or sys.argv[0] == '--resume': |
| 93 | if len(sys.argv) > 1: |
| 94 | resume_script=sys.argv[1] |
| 95 | sys.argv.pop(0) |
| 96 | sys.argv.pop(0) |
| 97 | else: |
| 98 | sys.argv.pop(0) |
| 99 | sys.stderr.write("Error: argument value required") |
| 100 | sys.exit(1) |
| 101 | elif sys.argv[0] == '--list': |
| 102 | sys.argv.pop(0) |
| 103 | for t in tests: |
| 104 | print t |
| 105 | sys.exit(0) |
| 106 | elif sys.argv[0] == '--list-xml': |
| 107 | sys.argv.pop(0) |
| 108 | for t in tests: |
| 109 | (mod,param) = t.split(None,2) |
| 110 | tname = mod[4:mod.find(".py")] + "_" + \ |
| 111 | param[param.find("/")+1:param.rfind(".")] |
| 112 | c = "" |
| 113 | if len(sys.argv): |
| 114 | c = " ".join(sys.argv) + " " |
| 115 | tcmd = PYTHON + ' run.py ' + c + t |
| 116 | print '\t\t<Test name="%s" cmd="%s" wdir="tests/pjsua" />' % (tname, tcmd) |
| 117 | sys.exit(0) |
| 118 | elif sys.argv[0] == '-s' or sys.argv[0] == '--shell': |
| 119 | if len(sys.argv) > 1: |
| 120 | shell_cmd = sys.argv[1] |
| 121 | sys.argv.pop(0) |
| 122 | sys.argv.pop(0) |
| 123 | else: |
| 124 | sys.argv.pop(0) |
| 125 | sys.stderr.write("Error: argument value required") |
| 126 | sys.exit(1) |
| 127 | else: |
| 128 | # should be run.py options |
| 129 | break |
| 130 | |
| 131 | |
| 132 | # Generate arguments for run.py |
| 133 | argv_st = " ".join(sys.argv) + " " |
| 134 | |
| 135 | # Init vars |
| 136 | fails_cnt = 0 |
| 137 | tests_cnt = 0 |
| 138 | |
| 139 | # Re-create "logs" directory |
| 140 | try: |
| 141 | shutil.rmtree("logs") |
| 142 | except: |
| 143 | print "Warning: failed in removing directory 'logs'" |
| 144 | |
| 145 | try: |
| 146 | os.mkdir("logs") |
| 147 | except: |
| 148 | print "Warning: failed in creating directory 'logs'" |
| 149 | |
| 150 | # Now run the tests |
| 151 | total_cnt = len(tests) |
| 152 | for t in tests: |
| 153 | if resume_script!="" and t.find(resume_script)==-1: |
| 154 | print "Skipping " + t +".." |
| 155 | total_cnt = total_cnt - 1 |
| 156 | continue |
| 157 | resume_script="" |
| 158 | cmdline = "python run.py " + argv_st + t |
| 159 | if shell_cmd: |
| 160 | cmdline = "%s '%s'" % (shell_cmd, cmdline) |
| 161 | t0 = time.time() |
| 162 | msg = "Running %d/%d: %s..." % (tests_cnt+1, total_cnt, cmdline) |
| 163 | sys.stdout.write(msg) |
| 164 | sys.stdout.flush() |
| 165 | ret = os.system(cmdline + " > output.log") |
| 166 | t1 = time.time() |
| 167 | if ret != 0: |
| 168 | dur = int(t1 - t0) |
| 169 | print " failed!! [" + str(dur) + "s]" |
| 170 | logname = re.search(".*\s+(.*)", t).group(1) |
| 171 | logname = re.sub("[\\\/]", "_", logname) |
| 172 | logname = re.sub("\.py$", ".log", logname) |
| 173 | logname = re.sub("\.xml$", ".log", logname) |
| 174 | logname = "logs/" + logname |
| 175 | shutil.move("output.log", logname) |
| 176 | print "Please see '" + logname + "' for the test log." |
| 177 | fails_cnt += 1 |
| 178 | else: |
| 179 | dur = int(t1 - t0) |
| 180 | print " ok [" + str(dur) + "s]" |
| 181 | tests_cnt += 1 |
| 182 | |
| 183 | if fails_cnt == 0: |
| 184 | print "All " + str(tests_cnt) + " tests completed successfully" |
| 185 | else: |
| 186 | print str(tests_cnt) + " tests completed, " + str(fails_cnt) + " test(s) failed" |
| 187 | |