1 /**
2    The different TestCase classes
3  */
4 module unit_threaded.runner.testcase;
5 
6 
7 private shared(bool) _stacktrace = false;
8 
9 private void setStackTrace(bool value) @trusted nothrow @nogc {
10     synchronized {
11         _stacktrace = value;
12     }
13 }
14 
15 /// Let AssertError(s) propagate and thus dump a stacktrace.
16 public void enableStackTrace() @safe nothrow @nogc {
17     setStackTrace(true);
18 }
19 
20 /// (Default behavior) Catch AssertError(s) and thus allow all tests to be ran.
21 public void disableStackTrace() @safe nothrow @nogc {
22     setStackTrace(false);
23 }
24 
25 /**
26  * Class from which other test cases derive
27  */
28 class TestCase {
29 
30     import unit_threaded.runner.io: Output;
31 
32     /**
33      * Returns: the name of the test
34      */
35     string getPath() const pure nothrow {
36         return this.classinfo.name;
37     }
38 
39     /**
40      * Executes the test.
41      * Returns: array of failures (child classes may have more than 1)
42      */
43     string[] opCall() {
44         static if(__VERSION__ >= 2077)
45             import std.datetime.stopwatch: StopWatch, AutoStart;
46         else
47             import std.datetime: StopWatch, AutoStart;
48 
49         currentTest = this;
50         auto sw = StopWatch(AutoStart.yes);
51         doTest();
52         flushOutput();
53         return _failed ? [getPath()] : [];
54     }
55 
56     /**
57      Certain child classes override this
58      */
59     ulong numTestsRun() const { return 1; }
60     void showChrono() @safe pure nothrow { _showChrono = true; }
61     void setOutput(Output output) @safe pure nothrow { _output = output; }
62     void silence() @safe pure nothrow { _silent = true; }
63     void quiet() @safe pure nothrow { _quiet = true; }
64     bool shouldFail() @safe @nogc pure nothrow { return false; }
65 
66 
67 package:
68 
69     static TestCase currentTest;
70     Output _output;
71 
72     final Output getWriter() @safe {
73         import unit_threaded.runner.io: WriterThread;
74         return _output is null ? WriterThread.get : _output;
75     }
76 
77 
78 protected:
79 
80     abstract void test();
81     void setup() { } ///override to run before test()
82     void shutdown() { } ///override to run after test()
83 
84 
85 private:
86 
87     bool _failed;
88     bool _silent;
89     bool _quiet;
90     bool _showChrono;
91 
92     final auto doTest() {
93         import std.conv: text;
94         import std.datetime: Duration;
95         static if(__VERSION__ >= 2077)
96             import std.datetime.stopwatch: StopWatch, AutoStart;
97         else
98             import std.datetime: StopWatch, AutoStart;
99 
100         auto sw = StopWatch(AutoStart.yes);
101         // Print the name of the test, unless in quiet mode.
102         // However, we want to print everything if it fails.
103         if(!_quiet) printTestName;
104         check(setup());
105         if (!_failed) check(test());
106         if (!_failed) check(shutdown());
107         if(_failed) print("\n");
108         if(_showChrono) print(text("    (", cast(Duration)sw.peek, ")\n\n"));
109         if(_failed) print("\n");
110     }
111 
112     final void printTestName() {
113         print(getPath() ~ ":\n");
114     }
115 
116     final bool check(E)(lazy E expression) {
117         import unit_threaded.exception: UnitTestException;
118         try {
119             expression();
120         } catch(UnitTestException ex) {
121             fail(ex.toString());
122         } catch(Throwable ex) {
123             fail("\n    " ~ ex.toString() ~ "\n");
124         }
125 
126         return !_failed;
127     }
128 
129     final void fail(in string msg) {
130         // if this is the first failure and in quiet mode, print the test
131         // name since we didn't do it at first
132         if(!_failed && _quiet) printTestName;
133         _failed = true;
134         print(msg);
135     }
136 
137     final void print(in string msg) {
138         import unit_threaded.runner.io: write;
139         if(!_silent) getWriter.write(msg);
140     }
141 
142     final void alwaysPrint(in string msg) {
143         import unit_threaded.runner.io: write;
144         getWriter.write(msg);
145     }
146 
147     final void flushOutput() {
148         getWriter.flush;
149     }
150 }
151 
152 unittest
153 {
154     enum Stage { setup, test, shutdown, none, }
155 
156     class TestForFailingStage : TestCase
157     {
158         Stage failedStage, currStage;
159 
160         this(Stage failedStage)
161         {
162             this.failedStage = failedStage;
163         }
164 
165         override void setup()
166         {
167             currStage = Stage.setup;
168             if (failedStage == currStage) assert(0);
169         }
170 
171         override void test()
172         {
173             currStage = Stage.test;
174             if (failedStage == currStage) assert(0);
175         }
176 
177         override void shutdown()
178         {
179             currStage = Stage.shutdown;
180             if (failedStage == currStage) assert(0);
181         }
182     }
183 
184     // the last stage of non failing test case is the shutdown stage
185     {
186         auto test = new TestForFailingStage(Stage.none);
187         test.silence;
188         test.doTest;
189 
190         assert(test.failedStage == Stage.none);
191         assert(test.currStage   == Stage.shutdown);
192     }
193 
194     // if a test case fails at setup stage the last stage is setup one
195     {
196         auto test = new TestForFailingStage(Stage.setup);
197         test.silence;
198         test.doTest;
199 
200         assert(test.failedStage == Stage.setup);
201         assert(test.currStage   == Stage.setup);
202     }
203 
204     // if a test case fails at test stage the last stage is test stage
205     {
206         auto test = new TestForFailingStage(Stage.test);
207         test.silence;
208         test.doTest;
209 
210         assert(test.failedStage == Stage.test);
211         assert(test.currStage   == Stage.test);
212     }
213 }
214 
215 /**
216    A test that runs other tests.
217  */
218 class CompositeTestCase: TestCase {
219     void add(TestCase t) @safe pure { _tests ~= t;}
220 
221     void opOpAssign(string op : "~")(TestCase t) {
222         add(t);
223     }
224 
225     override string[] opCall() {
226         import std.algorithm: map, reduce;
227         return _tests.map!(a => a()).reduce!((a, b) => a ~ b);
228     }
229 
230     override void test() { assert(false, "CompositeTestCase.test should never be called"); }
231 
232     override ulong numTestsRun() const {
233         return _tests.length;
234     }
235 
236     package TestCase[] tests() @safe pure nothrow {
237         return _tests;
238     }
239 
240     override void showChrono() {
241         foreach(test; _tests) test.showChrono;
242     }
243 
244 private:
245 
246     TestCase[] _tests;
247 }
248 
249 /**
250    A test that should fail
251  */
252 class ShouldFailTestCase: TestCase {
253     this(TestCase testCase, in TypeInfo exceptionTypeInfo) @safe pure {
254         this.testCase = testCase;
255         this.exceptionTypeInfo = exceptionTypeInfo;
256     }
257 
258     override bool shouldFail() @safe @nogc pure nothrow {
259         return true;
260     }
261 
262     override string getPath() const pure nothrow {
263         return this.testCase.getPath;
264     }
265 
266     override void test() {
267         import unit_threaded.exception: UnitTestException;
268         import std.exception: enforce, collectException;
269         import std.conv: text;
270 
271         const ex = collectException!Throwable(testCase.test());
272         enforce!UnitTestException(ex !is null, "Test '" ~ testCase.getPath ~ "' was expected to fail but did not");
273         enforce!UnitTestException(exceptionTypeInfo is null || typeid(ex) == exceptionTypeInfo,
274                                   text("Test '", testCase.getPath, "' was expected to throw ",
275                                        exceptionTypeInfo, " but threw ", typeid(ex)));
276     }
277 
278 private:
279 
280     TestCase testCase;
281     const(TypeInfo) exceptionTypeInfo;
282 }
283 
284 /**
285    A test that is a regular function.
286  */
287 class FunctionTestCase: TestCase {
288 
289     import unit_threaded.runner.reflection: TestData, TestFunction;
290 
291     this(in TestData data) @safe pure nothrow {
292         _name = data.getPath;
293         _func = data.testFunction;
294     }
295 
296     override void test() {
297         _func();
298     }
299 
300     override string getPath() const pure nothrow {
301         return _name;
302     }
303 
304     private string _name;
305     private TestFunction _func;
306 }
307 
308 /**
309    A test that is a `unittest` block.
310  */
311 class BuiltinTestCase: FunctionTestCase {
312 
313     import unit_threaded.runner.reflection: TestData;
314 
315     this(in TestData data) @safe pure nothrow {
316         super(data);
317     }
318 
319     override void test() {
320         import core.exception: AssertError;
321 
322         try
323             super.test();
324         catch(AssertError e) {
325             import unit_threaded.exception: fail;
326             // 3 = BuiltinTestCase + FunctionTestCase + runner reflection
327             fail(_stacktrace? e.toString() : e.localStacktraceToString(3), e.file, e.line);
328         }
329     }
330 }
331 
332 /**
333  * Generate `toString` text for a `Throwable` that contains just the stack trace
334  * below the current location, plus some additional number of trace lines.
335  *
336  * Used to generate a backtrace that cuts off exactly at a unittest body.
337  */
338 private string localStacktraceToString(Throwable throwable, int removeExtraLines) {
339     import std.algorithm: commonPrefix, count;
340     import std.range: dropBack, retro;
341 
342     // grab a stack trace inside this function
343     Throwable.TraceInfo localTraceInfo;
344     try throw new Exception("");
345     catch (Exception exc) localTraceInfo = exc.info;
346 
347     // convert foreach() overloads to arrays
348     string[] array(Throwable.TraceInfo info) {
349         string[] result;
350         foreach (line; info) result ~= line.idup;
351         return result;
352     }
353 
354     const string[] localBacktrace = array(localTraceInfo);
355     const string[] otherBacktrace = array(throwable.info);
356     // cut off shared lines of backtrace (plus some extra)
357     const size_t linesToRemove = otherBacktrace.retro.commonPrefix(localBacktrace.retro).count + removeExtraLines;
358     const string[] uniqueBacktrace = otherBacktrace.dropBack(linesToRemove);
359     // this should probably not be writable. ¯\_(ツ)_/¯
360     throwable.info = new class Throwable.TraceInfo {
361         override int opApply(scope int delegate(ref const(char[])) dg) const {
362             foreach (ref line; uniqueBacktrace)
363                 if (int ret = dg(line)) return ret;
364             return 0;
365         }
366         override int opApply(scope int delegate(ref size_t, ref const(char[])) dg) const {
367             foreach (ref i, ref line; uniqueBacktrace)
368                 if (int ret = dg(i, line)) return ret;
369             return 0;
370         }
371         override string toString() const { assert(false); }
372     };
373     return throwable.toString();
374 }
375 
376 unittest {
377     import std.conv : to;
378     import std..string : splitLines, indexOf;
379     import std.format : format;
380 
381     try throw new Exception("");
382     catch (Exception exc) {
383         const output = exc.localStacktraceToString(0);
384         const lines = output.splitLines;
385 
386         /*
387          * The text of a stacktrace can differ between compilers and also paths differ between Unix and Windows.
388          * Example exception test from dmd on unix:
389          *
390          * object.Exception@subpackages/runner/source/unit_threaded/runner/testcase.d(368)
391          * ----------------
392          * subpackages/runner/source/unit_threaded/runner/testcase.d:368 void unit_threaded.runner.testcase [...]
393          */
394         import std.stdio : writeln;
395         writeln("Output from local stack trace was " ~ to!string(lines.length) ~ " lines:\n"~output~"\n");
396 
397         assert(lines.length >= 3, "Expected 3 or more lines but got " ~ to!string(lines.length) ~ " :\n" ~ output);
398         assert(lines[0].indexOf("object.Exception@") != -1, "Line 1 of stack trace should show exception type. Was: "~lines[0]);
399 	    assert(lines[1].indexOf("------") != -1); // second line is a bunch of dashes
400         //assert(lines[2].indexOf("testcase.d") != -1); // the third line differs accross compilers and not reliable for testing
401     }
402 }
403 
404 /**
405    A test that is expected to fail some of the time.
406  */
407 class FlakyTestCase: TestCase {
408 
409     this(TestCase testCase, int retries) @safe pure {
410         this.testCase = testCase;
411         this.retries = retries;
412     }
413 
414     override string getPath() const pure nothrow {
415         return this.testCase.getPath;
416     }
417 
418     override void test() {
419 
420         foreach(i; 0 .. retries) {
421             try {
422                 testCase.test;
423                 break;
424             } catch(Throwable t) {
425                 if(i == retries - 1)
426                     throw t;
427             }
428         }
429     }
430 
431 private:
432 
433     TestCase testCase;
434     int retries;
435 }