runtests.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 
3 from __future__ import print_function
4 import argparse, glob, itertools, re, shutil, os, sys
5 
6 config_reg = re.compile('.*\/\/\s*(?P<name>\S+):\s*(?P<value>.*)$')
7 
8 class Tester:
9  def __init__(self,args,test):
10  self.args = args
11  self.test = test
12  self.update = args.updateref
13  self.config = self.get_config()
14  self.test_name = '[%s]: %s' % (self.test,self.config['objective'][0])
15  self.test_id = self.test.split('_')[0]
16  if self.update:
17  self.test_out = self.args.inputdir+'/'+self.test_id
18  else:
19  self.test_out = self.args.outputdir+'/test_output_'+self.test_id
20  self.prepare_test()
21 
22  def compare_ok(self,got_file,expected_file,name):
23  if not os.path.isfile(got_file):
24  return (True,'%s absent' % got_file)
25  elif not os.path.isfile(expected_file):
26  return (True,'%s absent' % expected_file)
27  else:
28  diff = os.popen('diff -b -w -u %s %s' % (got_file,expected_file)).read()
29  if diff and not diff.startswith("No differences"):
30  return (True,'Difference between generated output and reference:\n%s' % diff)
31  return (False,'')
32 
33  def get_config(self):
34  config = {}
35  with open(self.args.inputdir+'/'+self.test,'r') as f:
36  for line in f.readlines():
37  m = config_reg.match(line)
38  if m:
39  key = m.group('name')
40  value = m.group('value')
41  if (key=='config'):
42  value = value.replace('$INPUTDIR',self.args.inputdir)
43  #print('key=%s value=%s' % (key,value))
44  config.setdefault(key, []).append(value)
45  return config
46 
47  def prepare_test(self):
48  # prepare test environment
49  shutil.rmtree(self.test_out,ignore_errors=True)
50  os.mkdir(self.test_out)
51  shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out)
52  with open(self.test_out+'/Doxyfile','a') as f:
53  print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f)
54  print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f)
55  print('XML_OUTPUT=%s/out' % self.test_out, file=f)
56  print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f)
57  if 'config' in self.config:
58  for option in self.config['config']:
59  print(option, file=f)
60 
61  if 'check' not in self.config or not self.config['check']:
62  print('Test doesn\'t specify any files to check')
63  sys.exit(1)
64 
65  # run doxygen
66  if (sys.platform == 'win32'):
67  redir=' > nul:'
68  else:
69  redir=' 2> /dev/null'
70  if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out, redir))!=0:
71  print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out));
72  sys.exit(1)
73 
74  # update the reference data for this test
75  def update_test(self,testmgr):
76  print('Updating reference for %s' % self.test_name)
77 
78  if 'check' in self.config:
79  for check in self.config['check']:
80  check_file='%s/out/%s' % (self.test_out,check)
81  # check if the file we need to check is actually generated
82  if not os.path.isfile(check_file):
83  print('Non-existing file %s after \'check:\' statement' % check_file)
84  return
85  # convert output to canonical form
86  data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
87  if data:
88  # strip version
89  data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
90  else:
91  print('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
92  return
93  out_file='%s/%s' % (self.test_out,check)
94  with open(out_file,'w') as f:
95  print(data,file=f)
96  shutil.rmtree(self.test_out+'/out',ignore_errors=True)
97  os.remove(self.test_out+'/Doxyfile')
98 
99  # check the relevant files of a doxygen run with the reference material
100  def perform_test(self,testmgr):
101  # look for files to check against the reference
102  if 'check' in self.config:
103  for check in self.config['check']:
104  check_file='%s/out/%s' % (self.test_out,check)
105  # check if the file we need to check is actually generated
106  if not os.path.isfile(check_file):
107  testmgr.ok(False,self.test_name,msg='Non-existing file %s after \'check:\' statement' % check_file)
108  return
109  # convert output to canonical form
110  data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
111  if data:
112  # strip version
113  data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
114  else:
115  testmgr.ok(False,self.test_name,msg='Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
116  return
117  out_file='%s/%s' % (self.test_out,check)
118  with open(out_file,'w') as f:
119  print(data,file=f)
120  ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check)
121  (failed,msg) = self.compare_ok(out_file,ref_file,self.test_name)
122  if failed:
123  testmgr.ok(False,self.test_name,msg)
124  return
125  shutil.rmtree(self.test_out,ignore_errors=True)
126  testmgr.ok(True,self.test_name)
127 
128  def run(self,testmgr):
129  if self.update:
130  self.update_test(testmgr)
131  else:
132  self.perform_test(testmgr)
133 
135  def __init__(self,args,tests):
136  self.args = args
137  self.tests = tests
138  self.num_tests = len(tests)
139  self.count=1
140  self.passed=0
141  print('1..%d' % self.num_tests)
142 
143  def ok(self,result,test_name,msg='Ok'):
144  if result:
145  print('ok %s - %s' % (self.count,test_name))
146  self.passed = self.passed + 1
147  else:
148  print('not ok %s - %s' % (self.count,test_name))
149  print('-------------------------------------')
150  print(msg)
151  print('-------------------------------------')
152  self.count = self.count + 1
153 
154  def result(self):
155  if self.passed==self.num_tests:
156  print('All tests passed!')
157  else:
158  print('%d out of %s tests failed' % (self.num_tests-self.passed,self.num_tests))
159  return 0 if self.passed==self.num_tests else 1
160 
161  def perform_tests(self):
162  for test in self.tests:
163  tester = Tester(self.args,test)
164  tester.run(self)
165  return 0 if self.args.updateref else self.result()
166 
167 def main():
168  # argument handling
169  parser = argparse.ArgumentParser(description='run doxygen tests')
170  parser.add_argument('--updateref',help='update the reference data for a test',action="store_true")
171  parser.add_argument('--doxygen',nargs='?',default='doxygen',help='path/name of the doxygen executable')
172  parser.add_argument('--xmllint',nargs='?',default='xmllint',help='path/name of the xmllint executable')
173  parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help='id of the test to perform')
174  parser.add_argument('--all',help='perform all tests',action="store_true")
175  parser.add_argument('--inputdir',nargs='?',default='.',help='input directory containing the tests')
176  parser.add_argument('--outputdir',nargs='?',default='.',help='output directory to write the doxygen output to')
177  args = parser.parse_args()
178 
179  # sanity check
180  if (not args.updateref is None) and (args.ids is None) and (args.all is None):
181  parser.error('--updateref requires either --id or --all')
182 
183  starting_directory = os.getcwd()
184  os.chdir(args.inputdir)
185  # find the tests to run
186  if args.ids: # test ids are given by user
187  tests = []
188  for id in list(itertools.chain.from_iterable(args.ids)):
189  tests.append(glob.glob('%s_*'%id))
190  tests.append(glob.glob('0%s_*'%id))
191  tests.append(glob.glob('00%s_*'%id))
192  tests = list(itertools.chain.from_iterable(tests))
193  else: # find all tests
194  tests = glob.glob('[0-9][0-9][0-9]_*')
195  os.chdir(starting_directory)
196 
197  # create test manager to run the tests
198  testManager = TestManager(args,tests)
199  sys.exit(testManager.perform_tests())
200 
201 if __name__ == '__main__':
202  main()
def __init__(self, args, test)
Definition: runtests.py:9
def run(self, testmgr)
Definition: runtests.py:128
def main()
Definition: runtests.py:167
def ok(self, result, test_name, msg='Ok')
Definition: runtests.py:143
int open(const char *, int)
Opens a file descriptor.
def prepare_test(self)
Definition: runtests.py:47
def perform_tests(self)
Definition: runtests.py:161
def get_config(self)
Definition: runtests.py:33
def perform_test(self, testmgr)
Definition: runtests.py:100
def __init__(self, args, tests)
Definition: runtests.py:135
int read(int, char *, size_t)
Read bytes from a file descriptor.
def compare_ok(self, got_file, expected_file, name)
Definition: runtests.py:22
def update_test(self, testmgr)
Definition: runtests.py:75
def result(self)
Definition: runtests.py:154