1 /**
2    The different TestCase classes
3  */
4 module unit_threaded.runner.testcase;
5 
6 
7 private shared(bool) _stacktrace = false;
8 
9 private void setStackTrace(bool value) @trusted nothrow @nogc {
10     synchronized {
11         _stacktrace = value;
12     }
13 }
14 
15 /// Let AssertError(s) propagate and thus dump a stacktrace.
16 public void enableStackTrace() @safe nothrow @nogc {
17     setStackTrace(true);
18 }
19 
20 /// (Default behavior) Catch AssertError(s) and thus allow all tests to be ran.
21 public void disableStackTrace() @safe nothrow @nogc {
22     setStackTrace(false);
23 }
24 
25 /**
26  * Class from which other test cases derive
27  */
28 class TestCase {
29 
30     import unit_threaded.runner.io: Output;
31 
32     /**
33      * Returns: the name of the test
34      */
35     string getPath() const pure nothrow {
36         return this.classinfo.name;
37     }
38 
39     /**
40      * Executes the test.
41      * Returns: array of failures (child classes may have more than 1)
42      */
43     string[] opCall() {
44         currentTest = this;
45         doTest();
46         flushOutput();
47         return _failed ? [getPath()] : [];
48     }
49 
50     /**
51      Certain child classes override this
52      */
53     ulong numTestsRun() const { return 1; }
54     void showChrono() @safe pure nothrow { _showChrono = true; }
55     void setOutput(Output output) @safe pure nothrow { _output = output; }
56     void silence() @safe pure nothrow { _silent = true; }
57     void quiet() @safe pure nothrow { _quiet = true; }
58     bool shouldFail() @safe @nogc pure nothrow { return false; }
59 
60 
61 package(unit_threaded):
62 
63     static TestCase currentTest;
64     Output _output;
65 
66     final Output getWriter() @safe {
67         import unit_threaded.runner.io: WriterThread;
68         return _output is null ? WriterThread.get : _output;
69     }
70 
71 
72 protected:
73 
74     abstract void test();
75     void setup() { } ///override to run before test()
76     void shutdown() { } ///override to run after test()
77 
78 
79 private:
80 
81     bool _failed;
82     bool _silent;
83     bool _quiet;
84     bool _showChrono;
85 
86     final auto doTest() {
87         import std.conv: text;
88         import std.datetime: Duration;
89         static if(__VERSION__ >= 2077)
90             import std.datetime.stopwatch: StopWatch, AutoStart;
91         else
92             import std.datetime: StopWatch, AutoStart;
93 
94         auto sw = StopWatch(AutoStart.yes);
95         // Print the name of the test, unless in quiet mode.
96         // However, we want to print everything if it fails.
97         if(!_quiet) printTestName;
98         check(setup());
99         if (!_failed) check(test());
100         if (!_failed) check(shutdown());
101         if(_failed) print("\n");
102         if(_showChrono) print(text("    (", cast(Duration)sw.peek, ")\n\n"));
103         if(_failed) print("\n");
104     }
105 
106     final void printTestName() {
107         print(getPath() ~ ":\n");
108     }
109 
110     final bool check(E)(lazy E expression) {
111         import unit_threaded.exception: UnitTestError, UnitTestException;
112         try {
113             expression();
114         } catch(UnitTestException ex) {
115             fail(ex.toString());
116         } catch(UnitTestError err) {
117             fail(err.toString());
118         } catch(Throwable ex) {
119             fail("\n    " ~ ex.toString() ~ "\n");
120         }
121 
122         return !_failed;
123     }
124 
125     final void fail(in string msg) {
126         // if this is the first failure and in quiet mode, print the test
127         // name since we didn't do it at first
128         if(!_failed && _quiet) printTestName;
129         _failed = true;
130         print(msg);
131     }
132 
133     final void print(in string msg) {
134         import unit_threaded.runner.io: write;
135         if(!_silent) getWriter.write(msg);
136     }
137 
138     final void alwaysPrint(in string msg) {
139         import unit_threaded.runner.io: write;
140         getWriter.write(msg);
141     }
142 
143     final void flushOutput() {
144         getWriter.flush(!_failed);
145     }
146 }
147 
148 unittest
149 {
150     enum Stage { setup, test, shutdown, none, }
151 
152     class TestForFailingStage : TestCase
153     {
154         Stage failedStage, currStage;
155 
156         this(Stage failedStage)
157         {
158             this.failedStage = failedStage;
159         }
160 
161         override void setup()
162         {
163             currStage = Stage.setup;
164             if (failedStage == currStage) assert(0);
165         }
166 
167         override void test()
168         {
169             currStage = Stage.test;
170             if (failedStage == currStage) assert(0);
171         }
172 
173         override void shutdown()
174         {
175             currStage = Stage.shutdown;
176             if (failedStage == currStage) assert(0);
177         }
178     }
179 
180     // the last stage of non failing test case is the shutdown stage
181     {
182         auto test = new TestForFailingStage(Stage.none);
183         test.silence;
184         test.doTest;
185 
186         assert(test.failedStage == Stage.none);
187         assert(test.currStage   == Stage.shutdown);
188     }
189 
190     // if a test case fails at setup stage the last stage is setup one
191     {
192         auto test = new TestForFailingStage(Stage.setup);
193         test.silence;
194         test.doTest;
195 
196         assert(test.failedStage == Stage.setup);
197         assert(test.currStage   == Stage.setup);
198     }
199 
200     // if a test case fails at test stage the last stage is test stage
201     {
202         auto test = new TestForFailingStage(Stage.test);
203         test.silence;
204         test.doTest;
205 
206         assert(test.failedStage == Stage.test);
207         assert(test.currStage   == Stage.test);
208     }
209 }
210 
211 /**
212    A test that runs other tests.
213  */
214 class CompositeTestCase: TestCase {
215     void add(TestCase t) @safe pure { _tests ~= t;}
216 
217     void opOpAssign(string op : "~")(TestCase t) {
218         add(t);
219     }
220 
221     override string[] opCall() {
222         import std.algorithm: map, reduce;
223         return _tests.map!(a => a()).reduce!((a, b) => a ~ b);
224     }
225 
226     override void test() { assert(false, "CompositeTestCase.test should never be called"); }
227 
228     override ulong numTestsRun() const {
229         return _tests.length;
230     }
231 
232     package TestCase[] tests() @safe pure nothrow {
233         return _tests;
234     }
235 
236     override void showChrono() {
237         foreach(test; _tests) test.showChrono;
238     }
239 
240 private:
241 
242     TestCase[] _tests;
243 }
244 
245 /**
246    A test that should fail
247  */
248 class ShouldFailTestCase: TestCase {
249     this(TestCase testCase, in TypeInfo exceptionTypeInfo) @safe pure {
250         this.testCase = testCase;
251         this.exceptionTypeInfo = exceptionTypeInfo;
252     }
253 
254     override bool shouldFail() @safe @nogc pure nothrow {
255         return true;
256     }
257 
258     override string getPath() const pure nothrow {
259         return this.testCase.getPath;
260     }
261 
262     override void test() {
263         import unit_threaded.exception: UnitTestException;
264         import std.exception: enforce, collectException;
265         import std.conv: text;
266 
267         const ex = collectException!Throwable(testCase.test());
268         enforce!UnitTestException(ex !is null, "Test '" ~ testCase.getPath ~ "' was expected to fail but did not");
269         enforce!UnitTestException(exceptionTypeInfo is null || typeid(ex) == exceptionTypeInfo,
270                                   text("Test '", testCase.getPath, "' was expected to throw ",
271                                        exceptionTypeInfo, " but threw ", typeid(ex)));
272     }
273 
274 private:
275 
276     TestCase testCase;
277     const(TypeInfo) exceptionTypeInfo;
278 }
279 
280 /**
281    A test that is a regular function.
282  */
283 class FunctionTestCase: TestCase {
284 
285     import unit_threaded.runner.reflection: TestData, TestFunction;
286 
287     this(in TestData data) @safe pure nothrow {
288         _name = data.getPath;
289         _func = data.testFunction;
290     }
291 
292     override void test() {
293         _func();
294     }
295 
296     override string getPath() const pure nothrow {
297         return _name;
298     }
299 
300     private string _name;
301     private TestFunction _func;
302 }
303 
304 /**
305    A test that is a `unittest` block.
306  */
307 class BuiltinTestCase: FunctionTestCase {
308 
309     import unit_threaded.runner.reflection: TestData;
310 
311     this(in TestData data) @safe pure nothrow {
312         super(data);
313     }
314 
315     override void test() {
316         import core.exception: AssertError;
317 
318         try
319             super.test();
320         catch(AssertError e) {
321             import unit_threaded.exception: fail;
322             // 3 = BuiltinTestCase + FunctionTestCase + runner reflection
323             fail(_stacktrace? e.toString() : e.localStacktraceToString(3), e.file, e.line);
324         }
325     }
326 }
327 
328 /**
329  * Generate `toString` text for a `Throwable` that contains just the stack trace
330  * below the current location, plus some additional number of trace lines.
331  *
332  * Used to generate a backtrace that cuts off exactly at a unittest body.
333  */
334 private string localStacktraceToString(Throwable throwable, int removeExtraLines) {
335     import std.algorithm: commonPrefix, count;
336     import std.range: dropBack, retro;
337 
338     // grab a stack trace inside this function
339     Throwable.TraceInfo localTraceInfo;
340     try throw new Exception("");
341     catch (Exception exc) localTraceInfo = exc.info;
342 
343     // convert foreach() overloads to arrays
344     string[] array(Throwable.TraceInfo info) {
345         string[] result;
346         foreach (line; info) result ~= line.idup;
347         return result;
348     }
349 
350     const string[] localBacktrace = array(localTraceInfo);
351     const string[] otherBacktrace = array(throwable.info);
352     // cut off shared lines of backtrace (plus some extra)
353     const size_t linesToRemove = otherBacktrace.retro.commonPrefix(localBacktrace.retro).count + removeExtraLines;
354     const string[] uniqueBacktrace = otherBacktrace.dropBack(linesToRemove);
355     // this should probably not be writable. ¯\_(ツ)_/¯
356     throwable.info = new class Throwable.TraceInfo {
357         override int opApply(scope int delegate(ref const(char[])) dg) const {
358             foreach (ref line; uniqueBacktrace)
359                 if (int ret = dg(line)) return ret;
360             return 0;
361         }
362         override int opApply(scope int delegate(ref size_t, ref const(char[])) dg) const {
363             foreach (ref i, ref line; uniqueBacktrace)
364                 if (int ret = dg(i, line)) return ret;
365             return 0;
366         }
367         override string toString() const { assert(false); }
368     };
369     return throwable.toString();
370 }
371 
372 unittest {
373     import std.conv : to;
374     import std.string : splitLines, indexOf;
375     import std.format : format;
376 
377     try throw new Exception("");
378     catch (Exception exc) {
379         const output = exc.localStacktraceToString(0);
380         const lines = output.splitLines;
381 
382         /*
383          * The text of a stacktrace can differ between compilers and also paths differ between Unix and Windows.
384          * Example exception test from dmd on unix:
385          *
386          * object.Exception@subpackages/runner/source/unit_threaded/runner/testcase.d(368)
387          * ----------------
388          * subpackages/runner/source/unit_threaded/runner/testcase.d:368 void unit_threaded.runner.testcase [...]
389          */
390         import std.stdio : writeln;
391         writeln("Output from local stack trace was " ~ to!string(lines.length) ~ " lines:\n"~output~"\n");
392 
393         assert(lines.length >= 3, "Expected 3 or more lines but got " ~ to!string(lines.length) ~ " :\n" ~ output);
394         assert(lines[0].indexOf("object.Exception@") != -1, "Line 1 of stack trace should show exception type. Was: "~lines[0]);
395 	    assert(lines[1].indexOf("------") != -1); // second line is a bunch of dashes
396         //assert(lines[2].indexOf("testcase.d") != -1); // the third line differs accross compilers and not reliable for testing
397     }
398 }
399 
400 /**
401    A test that is expected to fail some of the time.
402  */
403 class FlakyTestCase: TestCase {
404 
405     this(TestCase testCase, int retries) @safe pure {
406         this.testCase = testCase;
407         this.retries = retries;
408     }
409 
410     override string getPath() const pure nothrow {
411         return this.testCase.getPath;
412     }
413 
414     override void test() {
415 
416         foreach(i; 0 .. retries) {
417             try {
418                 testCase.test;
419                 break;
420             } catch(Throwable t) {
421                 if(i == retries - 1)
422                     throw t;
423             }
424         }
425     }
426 
427 private:
428 
429     TestCase testCase;
430     int retries;
431 }