1 /** 2 The different TestCase classes 3 */ 4 module unit_threaded.runner.testcase; 5 6 7 private shared(bool) _stacktrace = false; 8 9 private void setStackTrace(bool value) @trusted nothrow @nogc { 10 synchronized { 11 _stacktrace = value; 12 } 13 } 14 15 /// Let AssertError(s) propagate and thus dump a stacktrace. 16 public void enableStackTrace() @safe nothrow @nogc { 17 setStackTrace(true); 18 } 19 20 /// (Default behavior) Catch AssertError(s) and thus allow all tests to be ran. 21 public void disableStackTrace() @safe nothrow @nogc { 22 setStackTrace(false); 23 } 24 25 /** 26 * Class from which other test cases derive 27 */ 28 class TestCase { 29 30 import unit_threaded.runner.io: Output; 31 32 /** 33 * Returns: the name of the test 34 */ 35 string getPath() const pure nothrow { 36 return this.classinfo.name; 37 } 38 39 /** 40 * Executes the test. 41 * Returns: array of failures (child classes may have more than 1) 42 */ 43 string[] opCall() { 44 static if(__VERSION__ >= 2077) 45 import std.datetime.stopwatch: StopWatch, AutoStart; 46 else 47 import std.datetime: StopWatch, AutoStart; 48 49 currentTest = this; 50 auto sw = StopWatch(AutoStart.yes); 51 doTest(); 52 flushOutput(); 53 return _failed ? [getPath()] : []; 54 } 55 56 /** 57 Certain child classes override this 58 */ 59 ulong numTestsRun() const { return 1; } 60 void showChrono() @safe pure nothrow { _showChrono = true; } 61 void setOutput(Output output) @safe pure nothrow { _output = output; } 62 void silence() @safe pure nothrow { _silent = true; } 63 void quiet() @safe pure nothrow { _quiet = true; } 64 bool shouldFail() @safe @nogc pure nothrow { return false; } 65 66 67 package: 68 69 static TestCase currentTest; 70 Output _output; 71 72 final Output getWriter() @safe { 73 import unit_threaded.runner.io: WriterThread; 74 return _output is null ? WriterThread.get : _output; 75 } 76 77 78 protected: 79 80 abstract void test(); 81 void setup() { } ///override to run before test() 82 void shutdown() { } ///override to run after test() 83 84 85 private: 86 87 bool _failed; 88 bool _silent; 89 bool _quiet; 90 bool _showChrono; 91 92 final auto doTest() { 93 import std.conv: text; 94 import std.datetime: Duration; 95 static if(__VERSION__ >= 2077) 96 import std.datetime.stopwatch: StopWatch, AutoStart; 97 else 98 import std.datetime: StopWatch, AutoStart; 99 100 auto sw = StopWatch(AutoStart.yes); 101 // Print the name of the test, unless in quiet mode. 102 // However, we want to print everything if it fails. 103 if(!_quiet) printTestName; 104 check(setup()); 105 if (!_failed) check(test()); 106 if (!_failed) check(shutdown()); 107 if(_failed) print("\n"); 108 if(_showChrono) print(text(" (", cast(Duration)sw.peek, ")\n\n")); 109 if(_failed) print("\n"); 110 } 111 112 final void printTestName() { 113 print(getPath() ~ ":\n"); 114 } 115 116 final bool check(E)(lazy E expression) { 117 import unit_threaded.exception: UnitTestException; 118 try { 119 expression(); 120 } catch(UnitTestException ex) { 121 fail(ex.toString()); 122 } catch(Throwable ex) { 123 fail("\n " ~ ex.toString() ~ "\n"); 124 } 125 126 return !_failed; 127 } 128 129 final void fail(in string msg) { 130 // if this is the first failure and in quiet mode, print the test 131 // name since we didn't do it at first 132 if(!_failed && _quiet) printTestName; 133 _failed = true; 134 print(msg); 135 } 136 137 final void print(in string msg) { 138 import unit_threaded.runner.io: write; 139 if(!_silent) getWriter.write(msg); 140 } 141 142 final void alwaysPrint(in string msg) { 143 import unit_threaded.runner.io: write; 144 getWriter.write(msg); 145 } 146 147 final void flushOutput() { 148 getWriter.flush; 149 } 150 } 151 152 unittest 153 { 154 enum Stage { setup, test, shutdown, none, } 155 156 class TestForFailingStage : TestCase 157 { 158 Stage failedStage, currStage; 159 160 this(Stage failedStage) 161 { 162 this.failedStage = failedStage; 163 } 164 165 override void setup() 166 { 167 currStage = Stage.setup; 168 if (failedStage == currStage) assert(0); 169 } 170 171 override void test() 172 { 173 currStage = Stage.test; 174 if (failedStage == currStage) assert(0); 175 } 176 177 override void shutdown() 178 { 179 currStage = Stage.shutdown; 180 if (failedStage == currStage) assert(0); 181 } 182 } 183 184 // the last stage of non failing test case is the shutdown stage 185 { 186 auto test = new TestForFailingStage(Stage.none); 187 test.silence; 188 test.doTest; 189 190 assert(test.failedStage == Stage.none); 191 assert(test.currStage == Stage.shutdown); 192 } 193 194 // if a test case fails at setup stage the last stage is setup one 195 { 196 auto test = new TestForFailingStage(Stage.setup); 197 test.silence; 198 test.doTest; 199 200 assert(test.failedStage == Stage.setup); 201 assert(test.currStage == Stage.setup); 202 } 203 204 // if a test case fails at test stage the last stage is test stage 205 { 206 auto test = new TestForFailingStage(Stage.test); 207 test.silence; 208 test.doTest; 209 210 assert(test.failedStage == Stage.test); 211 assert(test.currStage == Stage.test); 212 } 213 } 214 215 /** 216 A test that runs other tests. 217 */ 218 class CompositeTestCase: TestCase { 219 void add(TestCase t) @safe pure { _tests ~= t;} 220 221 void opOpAssign(string op : "~")(TestCase t) { 222 add(t); 223 } 224 225 override string[] opCall() { 226 import std.algorithm: map, reduce; 227 return _tests.map!(a => a()).reduce!((a, b) => a ~ b); 228 } 229 230 override void test() { assert(false, "CompositeTestCase.test should never be called"); } 231 232 override ulong numTestsRun() const { 233 return _tests.length; 234 } 235 236 package TestCase[] tests() @safe pure nothrow { 237 return _tests; 238 } 239 240 override void showChrono() { 241 foreach(test; _tests) test.showChrono; 242 } 243 244 private: 245 246 TestCase[] _tests; 247 } 248 249 /** 250 A test that should fail 251 */ 252 class ShouldFailTestCase: TestCase { 253 this(TestCase testCase, in TypeInfo exceptionTypeInfo) @safe pure { 254 this.testCase = testCase; 255 this.exceptionTypeInfo = exceptionTypeInfo; 256 } 257 258 override bool shouldFail() @safe @nogc pure nothrow { 259 return true; 260 } 261 262 override string getPath() const pure nothrow { 263 return this.testCase.getPath; 264 } 265 266 override void test() { 267 import unit_threaded.exception: UnitTestException; 268 import std.exception: enforce, collectException; 269 import std.conv: text; 270 271 const ex = collectException!Throwable(testCase.test()); 272 enforce!UnitTestException(ex !is null, "Test '" ~ testCase.getPath ~ "' was expected to fail but did not"); 273 enforce!UnitTestException(exceptionTypeInfo is null || typeid(ex) == exceptionTypeInfo, 274 text("Test '", testCase.getPath, "' was expected to throw ", 275 exceptionTypeInfo, " but threw ", typeid(ex))); 276 } 277 278 private: 279 280 TestCase testCase; 281 const(TypeInfo) exceptionTypeInfo; 282 } 283 284 /** 285 A test that is a regular function. 286 */ 287 class FunctionTestCase: TestCase { 288 289 import unit_threaded.runner.reflection: TestData, TestFunction; 290 291 this(in TestData data) @safe pure nothrow { 292 _name = data.getPath; 293 _func = data.testFunction; 294 } 295 296 override void test() { 297 _func(); 298 } 299 300 override string getPath() const pure nothrow { 301 return _name; 302 } 303 304 private string _name; 305 private TestFunction _func; 306 } 307 308 /** 309 A test that is a `unittest` block. 310 */ 311 class BuiltinTestCase: FunctionTestCase { 312 313 import unit_threaded.runner.reflection: TestData; 314 315 this(in TestData data) @safe pure nothrow { 316 super(data); 317 } 318 319 override void test() { 320 import core.exception: AssertError; 321 322 try 323 super.test(); 324 catch(AssertError e) { 325 import unit_threaded.exception: fail; 326 // 3 = BuiltinTestCase + FunctionTestCase + runner reflection 327 fail(_stacktrace? e.toString() : e.localStacktraceToString(3), e.file, e.line); 328 } 329 } 330 } 331 332 /** 333 * Generate `toString` text for a `Throwable` that contains just the stack trace 334 * below the current location, plus some additional number of trace lines. 335 * 336 * Used to generate a backtrace that cuts off exactly at a unittest body. 337 */ 338 private string localStacktraceToString(Throwable throwable, int removeExtraLines) { 339 import std.algorithm: commonPrefix, count; 340 import std.range: dropBack, retro; 341 342 // grab a stack trace inside this function 343 Throwable.TraceInfo localTraceInfo; 344 try throw new Exception(""); 345 catch (Exception exc) localTraceInfo = exc.info; 346 347 // convert foreach() overloads to arrays 348 string[] array(Throwable.TraceInfo info) { 349 string[] result; 350 foreach (line; info) result ~= line.idup; 351 return result; 352 } 353 354 const string[] localBacktrace = array(localTraceInfo); 355 const string[] otherBacktrace = array(throwable.info); 356 // cut off shared lines of backtrace (plus some extra) 357 const size_t linesToRemove = otherBacktrace.retro.commonPrefix(localBacktrace.retro).count + removeExtraLines; 358 const string[] uniqueBacktrace = otherBacktrace.dropBack(linesToRemove); 359 // this should probably not be writable. ¯\_(ツ)_/¯ 360 throwable.info = new class Throwable.TraceInfo { 361 override int opApply(scope int delegate(ref const(char[])) dg) const { 362 foreach (ref line; uniqueBacktrace) 363 if (int ret = dg(line)) return ret; 364 return 0; 365 } 366 override int opApply(scope int delegate(ref size_t, ref const(char[])) dg) const { 367 foreach (ref i, ref line; uniqueBacktrace) 368 if (int ret = dg(i, line)) return ret; 369 return 0; 370 } 371 override string toString() const { assert(false); } 372 }; 373 return throwable.toString(); 374 } 375 376 unittest { 377 import std.conv : to; 378 import std..string : splitLines, indexOf; 379 import std.format : format; 380 381 try throw new Exception(""); 382 catch (Exception exc) { 383 const output = exc.localStacktraceToString(0); 384 const lines = output.splitLines; 385 386 /* 387 * The text of a stacktrace can differ between compilers and also paths differ between Unix and Windows. 388 * Example exception test from dmd on unix: 389 * 390 * object.Exception@subpackages/runner/source/unit_threaded/runner/testcase.d(368) 391 * ---------------- 392 * subpackages/runner/source/unit_threaded/runner/testcase.d:368 void unit_threaded.runner.testcase [...] 393 */ 394 import std.stdio : writeln; 395 writeln("Output from local stack trace was " ~ to!string(lines.length) ~ " lines:\n"~output~"\n"); 396 397 assert(lines.length >= 3, "Expected 3 or more lines but got " ~ to!string(lines.length) ~ " :\n" ~ output); 398 assert(lines[0].indexOf("object.Exception@") != -1, "Line 1 of stack trace should show exception type. Was: "~lines[0]); 399 assert(lines[1].indexOf("------") != -1); // second line is a bunch of dashes 400 //assert(lines[2].indexOf("testcase.d") != -1); // the third line differs accross compilers and not reliable for testing 401 } 402 } 403 404 /** 405 A test that is expected to fail some of the time. 406 */ 407 class FlakyTestCase: TestCase { 408 409 this(TestCase testCase, int retries) @safe pure { 410 this.testCase = testCase; 411 this.retries = retries; 412 } 413 414 override string getPath() const pure nothrow { 415 return this.testCase.getPath; 416 } 417 418 override void test() { 419 420 foreach(i; 0 .. retries) { 421 try { 422 testCase.test; 423 break; 424 } catch(Throwable t) { 425 if(i == retries - 1) 426 throw t; 427 } 428 } 429 } 430 431 private: 432 433 TestCase testCase; 434 int retries; 435 }