1 /** 2 The different TestCase classes 3 */ 4 module unit_threaded.runner.testcase; 5 6 7 private shared(bool) _stacktrace = false; 8 9 private void setStackTrace(bool value) @trusted nothrow @nogc { 10 synchronized { 11 _stacktrace = value; 12 } 13 } 14 15 /// Let AssertError(s) propagate and thus dump a stacktrace. 16 public void enableStackTrace() @safe nothrow @nogc { 17 setStackTrace(true); 18 } 19 20 /// (Default behavior) Catch AssertError(s) and thus allow all tests to be ran. 21 public void disableStackTrace() @safe nothrow @nogc { 22 setStackTrace(false); 23 } 24 25 /** 26 * Class from which other test cases derive 27 */ 28 class TestCase { 29 30 import unit_threaded.runner.io: Output; 31 32 /** 33 * Returns: the name of the test 34 */ 35 string getPath() const pure nothrow { 36 return this.classinfo.name; 37 } 38 39 /** 40 * Executes the test. 41 * Returns: array of failures (child classes may have more than 1) 42 */ 43 string[] opCall() { 44 currentTest = this; 45 doTest(); 46 flushOutput(); 47 return _failed ? [getPath()] : []; 48 } 49 50 /** 51 Certain child classes override this 52 */ 53 ulong numTestsRun() const { return 1; } 54 void showChrono() @safe pure nothrow { _showChrono = true; } 55 void setOutput(Output output) @safe pure nothrow { _output = output; } 56 void silence() @safe pure nothrow { _silent = true; } 57 void quiet() @safe pure nothrow { _quiet = true; } 58 bool shouldFail() @safe @nogc pure nothrow { return false; } 59 60 61 package(unit_threaded): 62 63 static TestCase currentTest; 64 Output _output; 65 66 final Output getWriter() @safe { 67 import unit_threaded.runner.io: WriterThread; 68 return _output is null ? WriterThread.get : _output; 69 } 70 71 72 protected: 73 74 abstract void test(); 75 void setup() { } ///override to run before test() 76 void shutdown() { } ///override to run after test() 77 78 79 private: 80 81 bool _failed; 82 bool _silent; 83 bool _quiet; 84 bool _showChrono; 85 86 final auto doTest() { 87 import std.conv: text; 88 import std.datetime: Duration; 89 static if(__VERSION__ >= 2077) 90 import std.datetime.stopwatch: StopWatch, AutoStart; 91 else 92 import std.datetime: StopWatch, AutoStart; 93 94 auto sw = StopWatch(AutoStart.yes); 95 // Print the name of the test, unless in quiet mode. 96 // However, we want to print everything if it fails. 97 if(!_quiet) printTestName; 98 check(setup()); 99 if (!_failed) check(test()); 100 if (!_failed) check(shutdown()); 101 if(_failed) print("\n"); 102 if(_showChrono) print(text(" (", cast(Duration)sw.peek, ")\n\n")); 103 if(_failed) print("\n"); 104 } 105 106 final void printTestName() { 107 print(getPath() ~ ":\n"); 108 } 109 110 final bool check(E)(lazy E expression) { 111 import unit_threaded.exception: UnitTestError, UnitTestException; 112 try { 113 expression(); 114 } catch(UnitTestException ex) { 115 fail(ex.toString()); 116 } catch(UnitTestError err) { 117 fail(err.toString()); 118 } catch(Throwable ex) { 119 fail("\n " ~ ex.toString() ~ "\n"); 120 } 121 122 return !_failed; 123 } 124 125 final void fail(in string msg) { 126 // if this is the first failure and in quiet mode, print the test 127 // name since we didn't do it at first 128 if(!_failed && _quiet) printTestName; 129 _failed = true; 130 print(msg); 131 } 132 133 final void print(in string msg) { 134 import unit_threaded.runner.io: write; 135 if(!_silent) getWriter.write(msg); 136 } 137 138 final void alwaysPrint(in string msg) { 139 import unit_threaded.runner.io: write; 140 getWriter.write(msg); 141 } 142 143 final void flushOutput() { 144 getWriter.flush(!_failed); 145 } 146 } 147 148 unittest 149 { 150 enum Stage { setup, test, shutdown, none, } 151 152 class TestForFailingStage : TestCase 153 { 154 Stage failedStage, currStage; 155 156 this(Stage failedStage) 157 { 158 this.failedStage = failedStage; 159 } 160 161 override void setup() 162 { 163 currStage = Stage.setup; 164 if (failedStage == currStage) assert(0); 165 } 166 167 override void test() 168 { 169 currStage = Stage.test; 170 if (failedStage == currStage) assert(0); 171 } 172 173 override void shutdown() 174 { 175 currStage = Stage.shutdown; 176 if (failedStage == currStage) assert(0); 177 } 178 } 179 180 // the last stage of non failing test case is the shutdown stage 181 { 182 auto test = new TestForFailingStage(Stage.none); 183 test.silence; 184 test.doTest; 185 186 assert(test.failedStage == Stage.none); 187 assert(test.currStage == Stage.shutdown); 188 } 189 190 // if a test case fails at setup stage the last stage is setup one 191 { 192 auto test = new TestForFailingStage(Stage.setup); 193 test.silence; 194 test.doTest; 195 196 assert(test.failedStage == Stage.setup); 197 assert(test.currStage == Stage.setup); 198 } 199 200 // if a test case fails at test stage the last stage is test stage 201 { 202 auto test = new TestForFailingStage(Stage.test); 203 test.silence; 204 test.doTest; 205 206 assert(test.failedStage == Stage.test); 207 assert(test.currStage == Stage.test); 208 } 209 } 210 211 /** 212 A test that runs other tests. 213 */ 214 class CompositeTestCase: TestCase { 215 void add(TestCase t) @safe pure { _tests ~= t;} 216 217 void opOpAssign(string op : "~")(TestCase t) { 218 add(t); 219 } 220 221 override string[] opCall() { 222 import std.algorithm: map, reduce; 223 return _tests.map!(a => a()).reduce!((a, b) => a ~ b); 224 } 225 226 override void test() { assert(false, "CompositeTestCase.test should never be called"); } 227 228 override ulong numTestsRun() const { 229 return _tests.length; 230 } 231 232 package TestCase[] tests() @safe pure nothrow { 233 return _tests; 234 } 235 236 override void showChrono() { 237 foreach(test; _tests) test.showChrono; 238 } 239 240 private: 241 242 TestCase[] _tests; 243 } 244 245 /** 246 A test that should fail 247 */ 248 class ShouldFailTestCase: TestCase { 249 this(TestCase testCase, in TypeInfo exceptionTypeInfo) @safe pure { 250 this.testCase = testCase; 251 this.exceptionTypeInfo = exceptionTypeInfo; 252 } 253 254 override bool shouldFail() @safe @nogc pure nothrow { 255 return true; 256 } 257 258 override string getPath() const pure nothrow { 259 return this.testCase.getPath; 260 } 261 262 override void test() { 263 import unit_threaded.exception: UnitTestException; 264 import std.exception: enforce, collectException; 265 import std.conv: text; 266 267 const ex = collectException!Throwable(testCase.test()); 268 enforce!UnitTestException(ex !is null, "Test '" ~ testCase.getPath ~ "' was expected to fail but did not"); 269 enforce!UnitTestException(exceptionTypeInfo is null || typeid(ex) == exceptionTypeInfo, 270 text("Test '", testCase.getPath, "' was expected to throw ", 271 exceptionTypeInfo, " but threw ", typeid(ex))); 272 } 273 274 private: 275 276 TestCase testCase; 277 const(TypeInfo) exceptionTypeInfo; 278 } 279 280 /** 281 A test that is a regular function. 282 */ 283 class FunctionTestCase: TestCase { 284 285 import unit_threaded.runner.reflection: TestData, TestFunction; 286 287 this(in TestData data) @safe pure nothrow { 288 _name = data.getPath; 289 _func = data.testFunction; 290 } 291 292 override void test() { 293 _func(); 294 } 295 296 override string getPath() const pure nothrow { 297 return _name; 298 } 299 300 private string _name; 301 private TestFunction _func; 302 } 303 304 /** 305 A test that is a `unittest` block. 306 */ 307 class BuiltinTestCase: FunctionTestCase { 308 309 import unit_threaded.runner.reflection: TestData; 310 311 this(in TestData data) @safe pure nothrow { 312 super(data); 313 } 314 315 override void test() { 316 import core.exception: AssertError; 317 318 try 319 super.test(); 320 catch(AssertError e) { 321 import unit_threaded.exception: fail; 322 // 3 = BuiltinTestCase + FunctionTestCase + runner reflection 323 fail(_stacktrace? e.toString() : e.localStacktraceToString(3), e.file, e.line); 324 } 325 } 326 } 327 328 /** 329 * Generate `toString` text for a `Throwable` that contains just the stack trace 330 * below the current location, plus some additional number of trace lines. 331 * 332 * Used to generate a backtrace that cuts off exactly at a unittest body. 333 */ 334 private string localStacktraceToString(Throwable throwable, int removeExtraLines) { 335 import std.algorithm: commonPrefix, count; 336 import std.range: dropBack, retro; 337 338 // grab a stack trace inside this function 339 Throwable.TraceInfo localTraceInfo; 340 try throw new Exception(""); 341 catch (Exception exc) localTraceInfo = exc.info; 342 343 // convert foreach() overloads to arrays 344 string[] array(Throwable.TraceInfo info) { 345 string[] result; 346 foreach (line; info) result ~= line.idup; 347 return result; 348 } 349 350 const string[] localBacktrace = array(localTraceInfo); 351 const string[] otherBacktrace = array(throwable.info); 352 // cut off shared lines of backtrace (plus some extra) 353 const size_t linesToRemove = otherBacktrace.retro.commonPrefix(localBacktrace.retro).count + removeExtraLines; 354 const string[] uniqueBacktrace = otherBacktrace.dropBack(linesToRemove); 355 // this should probably not be writable. ¯\_(ツ)_/¯ 356 throwable.info = new class Throwable.TraceInfo { 357 override int opApply(scope int delegate(ref const(char[])) dg) const { 358 foreach (ref line; uniqueBacktrace) 359 if (int ret = dg(line)) return ret; 360 return 0; 361 } 362 override int opApply(scope int delegate(ref size_t, ref const(char[])) dg) const { 363 foreach (ref i, ref line; uniqueBacktrace) 364 if (int ret = dg(i, line)) return ret; 365 return 0; 366 } 367 override string toString() const { assert(false); } 368 }; 369 return throwable.toString(); 370 } 371 372 unittest { 373 import std.conv : to; 374 import std.string : splitLines, indexOf; 375 import std.format : format; 376 377 try throw new Exception(""); 378 catch (Exception exc) { 379 const output = exc.localStacktraceToString(0); 380 const lines = output.splitLines; 381 382 /* 383 * The text of a stacktrace can differ between compilers and also paths differ between Unix and Windows. 384 * Example exception test from dmd on unix: 385 * 386 * object.Exception@subpackages/runner/source/unit_threaded/runner/testcase.d(368) 387 * ---------------- 388 * subpackages/runner/source/unit_threaded/runner/testcase.d:368 void unit_threaded.runner.testcase [...] 389 */ 390 import std.stdio : writeln; 391 writeln("Output from local stack trace was " ~ to!string(lines.length) ~ " lines:\n"~output~"\n"); 392 393 assert(lines.length >= 3, "Expected 3 or more lines but got " ~ to!string(lines.length) ~ " :\n" ~ output); 394 assert(lines[0].indexOf("object.Exception@") != -1, "Line 1 of stack trace should show exception type. Was: "~lines[0]); 395 assert(lines[1].indexOf("------") != -1); // second line is a bunch of dashes 396 //assert(lines[2].indexOf("testcase.d") != -1); // the third line differs accross compilers and not reliable for testing 397 } 398 } 399 400 /** 401 A test that is expected to fail some of the time. 402 */ 403 class FlakyTestCase: TestCase { 404 405 this(TestCase testCase, int retries) @safe pure { 406 this.testCase = testCase; 407 this.retries = retries; 408 } 409 410 override string getPath() const pure nothrow { 411 return this.testCase.getPath; 412 } 413 414 override void test() { 415 416 foreach(i; 0 .. retries) { 417 try { 418 testCase.test; 419 break; 420 } catch(Throwable t) { 421 if(i == retries - 1) 422 throw t; 423 } 424 } 425 } 426 427 private: 428 429 TestCase testCase; 430 int retries; 431 }