1 /**
2  * This module implements $(D TestSuite), an aggregator for $(D TestCase)
3  * objects to run all tests.
4  */
5 
6 module unit_threaded.testsuite;
7 
8 import unit_threaded.testcase;
9 import unit_threaded.io;
10 import unit_threaded.options;
11 import std.datetime;
12 import std.parallelism : taskPool;
13 import std.algorithm;
14 import std.conv : text;
15 import std.array;
16 import core.runtime;
17 
18 /*
19  * taskPool.amap only works with public functions, not closures.
20  */
21 auto runTest(TestCase test)
22 {
23     return test();
24 }
25 
26 /**
27  * Responsible for running tests and printing output.
28  */
29 struct TestSuite
30 {
31     /**
32      * Params:
33      * options = The options to run tests with.
34      * testData = The information about the tests to run.
35      */
36     this(in Options options, in TestData[] testData)
37     {
38         _options = options;
39         _testData = testData;
40         _testCases = createTestCases(testData, options.testsToRun);
41         WriterThread.start;
42     }
43 
44     ~this()
45     {
46         WriterThread.get.join;
47     }
48 
49     /**
50      * Runs all test cases.
51      * Returns: true if no test failed, false otherwise.
52      */
53     bool run()
54     {
55         if (!_testCases.length)
56         {
57             utWritelnRed("Error! No tests to run for args: ");
58             utWriteln(_options.testsToRun);
59             return false;
60         }
61 
62         immutable elapsed = doRun();
63 
64         if (!numTestsRun)
65         {
66             utWriteln("Did not run any tests!!!");
67             return false;
68         }
69 
70         utWriteln("\nTime taken: ", elapsed);
71         utWrite(numTestsRun, " test(s) run, ");
72         const failuresStr = text(_failures.length, " failed");
73         if (_failures.length)
74         {
75             utWriteRed(failuresStr);
76         }
77         else
78         {
79             utWrite(failuresStr);
80         }
81 
82         void printAbout(string attr)(in string msg)
83         {
84             const num = _testData.filter!(a => mixin("a. " ~ attr)).count;
85             if (num)
86             {
87                 utWrite(", ");
88                 utWriteYellow(num, " " ~ msg);
89             }
90         }
91 
92         printAbout!"hidden"("hidden");
93         printAbout!"shouldFail"("failing as expected");
94 
95         utWriteln(".\n");
96 
97         if (_failures.length)
98         {
99             utWritelnRed("Unit tests failed!\n");
100             return false; //oops
101         }
102 
103         utWritelnGreen("OK!\n");
104 
105         return true;
106     }
107 
108 private:
109 
110     const(Options) _options;
111     const(TestData)[] _testData;
112     TestCase[] _testCases;
113     string[] _failures;
114     StopWatch _stopWatch;
115 
116     /**
117      * Runs the tests with the given options.
118      * Returns: how long it took to run.
119      */
120     Duration doRun()
121     {
122         auto tests = getTests();
123         _stopWatch.start();
124 
125         if (_options.multiThreaded)
126         {
127             _failures = reduce!((a, b) => a ~ b)(_failures, taskPool.amap!runTest(tests));
128         }
129         else
130         {
131             foreach (test; tests)
132                 _failures ~= test();
133         }
134 
135         handleFailures();
136 
137         _stopWatch.stop();
138         return cast(Duration) _stopWatch.peek();
139     }
140 
141     auto getTests()
142     {
143         auto tests = _testCases.dup;
144         if (_options.random)
145         {
146             import std.random;
147 
148             auto generator = Random(_options.seed);
149             tests.randomShuffle(generator);
150             utWriteln("Running tests in random order. ",
151                 "To repeat this run, use --seed ", _options.seed);
152         }
153         return tests;
154     }
155 
156     void handleFailures() const
157     {
158         if (!_failures.empty)
159             utWriteln("");
160         foreach (failure; _failures)
161         {
162             utWrite("Test ", failure, " ");
163             utWriteRed("failed");
164             utWriteln(".");
165         }
166         if (!_failures.empty)
167             utWriteln("");
168     }
169 
170     @property ulong numTestsRun() @safe const pure
171     {
172         return _testCases.map!(a => a.numTestsRun).reduce!((a, b) => a + b);
173     }
174 }
175 
176 /**
177  * Replace the D runtime's normal unittest block tester. If this is not done,
178  * the tests will run twice.
179  */
180 void replaceModuleUnitTester()
181 {
182     import core.runtime;
183 
184     Runtime.moduleUnitTester = &moduleUnitTester;
185 }
186 
187 shared static this()
188 {
189     replaceModuleUnitTester();
190 }
191 
192 /**
193  * Replacement for the usual unittest runner. Since unit_threaded
194  * runs the tests itself, the moduleUnitTester doesn't have to do anything.
195  */
196 private bool moduleUnitTester()
197 {
198     return true;
199 }
200 
201 /**
202  * Creates tests cases from the given modules.
203  * If testsToRun is empty, it means run all tests.
204  */
205 private TestCase[] createTestCases(in TestData[] testData, in string[] testsToRun = []) @safe
206 {
207     bool[TestCase] tests;
208 
209     foreach (const data; testData)
210     {
211         if (!isWantedTest(data, testsToRun))
212             continue;
213         tests[createTestCase(data)] = true;
214     }
215 
216     return () @trusted{ return tests.keys.sort!((a, b) => a.name < b.name).array; }();
217 }
218 
219 private TestCase createTestCase(in TestData testData) @safe
220 {
221     auto testCase = new FunctionTestCase(testData);
222 
223     if (testData.serial)
224     {
225         // @serial tests in the same module run sequentially.
226         // A CompositeTestCase is created for each module with at least
227         // one @serial test and subsequent @serial tests
228         // appended to it
229         static CompositeTestCase[string] composites;
230 
231         const moduleName = testData.name.splitter(".").array[0 .. $ - 1].reduce!((a,
232             b) => a ~ "." ~ b);
233 
234         if (moduleName !in composites)
235             composites[moduleName] = new CompositeTestCase;
236 
237         composites[moduleName] ~= testCase;
238         return composites[moduleName];
239     }
240 
241     if (testData.shouldFail)
242     {
243         return new ShouldFailTestCase(testCase);
244     }
245 
246     return testCase;
247 }
248 
249 private bool isWantedTest(in TestData testData, in string[] testsToRun) @safe pure
250 {
251     //hidden tests are not run by default, every other one is
252     if (!testsToRun.length)
253         return !testData.hidden;
254     bool matchesExactly(in string t)
255     {
256         return t == testData.name;
257     }
258 
259     bool matchesPackage(in string t) //runs all tests in package if it matches
260     {
261         with (testData)
262             return !hidden && name.length > t.length && name.startsWith(t)
263                 && name[t.length .. $].canFind(".");
264     }
265 
266     return testsToRun.any!(t => matchesExactly(t) || matchesPackage(t));
267 }
268 
269 unittest
270 {
271     //existing, wanted
272     assert(isWantedTest(TestData("tests.server.testSubscribe"), ["tests"]));
273     assert(isWantedTest(TestData("tests.server.testSubscribe"), ["tests."]));
274     assert(isWantedTest(TestData("tests.server.testSubscribe"), ["tests.server.testSubscribe"]));
275     assert(!isWantedTest(TestData("tests.server.testSubscribe"),
276         ["tests.server.testSubscribeWithMessage"]));
277     assert(!isWantedTest(TestData("tests.stream.testMqttInTwoPackets"), ["tests.server"]));
278     assert(isWantedTest(TestData("tests.server.testSubscribe"), ["tests.server"]));
279     assert(isWantedTest(TestData("pass_tests.testEqual"), ["pass_tests"]));
280     assert(isWantedTest(TestData("pass_tests.testEqual"), ["pass_tests.testEqual"]));
281     assert(isWantedTest(TestData("pass_tests.testEqual"), []));
282     assert(!isWantedTest(TestData("pass_tests.testEqual"), ["pass_tests.foo"]));
283     assert(!isWantedTest(TestData("example.tests.pass.normal.unittest"),
284         ["example.tests.pass.io.TestFoo"]));
285     assert(isWantedTest(TestData("example.tests.pass.normal.unittest"), []));
286     assert(!isWantedTest(TestData("tests.pass.attributes.testHidden", null  /*func*/ ,
287         true  /*hidden*/ ), ["tests.pass"]));
288 }