1 module unit_threaded.reflection;
2 
3 import unit_threaded.attrs;
4 import unit_threaded.uda;
5 import unit_threaded.meta;
6 import std.traits;
7 import std.typetuple;
8 
9 /**
10  * Common data for test functions and test classes
11  */
12 alias void delegate() TestFunction;
13 struct TestData {
14     string name;
15     TestFunction testFunction; ///only used for functions, null for classes
16     bool hidden;
17     bool shouldFail;
18     bool singleThreaded;
19     bool builtin;
20     string suffix; // append to end of getPath
21 
22     string getPath() const pure nothrow {
23         string path = name.dup;
24         import std.array: empty;
25         if(!suffix.empty) path ~= "." ~ suffix;
26         return path;
27     }
28 
29     bool isTestClass() @safe const pure nothrow {
30         return testFunction is null;
31     }
32 }
33 
34 
35 /**
36  * Finds all test cases (functions, classes, built-in unittest blocks)
37  * Template parameters are module strings
38  */
39 const(TestData)[] allTestData(MOD_STRINGS...)() if(allSatisfy!(isSomeString, typeof(MOD_STRINGS))) {
40 
41     string getModulesString() {
42         import std.array: join;
43         string[] modules;
44         foreach(module_; MOD_STRINGS) modules ~= module_;
45         return modules.join(", ");
46     }
47 
48     enum modulesString =  getModulesString;
49     mixin("import " ~ modulesString ~ ";");
50     mixin("return allTestData!(" ~ modulesString ~ ");");
51 }
52 
53 
54 /**
55  * Finds all test cases (functions, classes, built-in unittest blocks)
56  * Template parameters are module symbols
57  */
58 const(TestData)[] allTestData(MOD_SYMBOLS...)() if(!anySatisfy!(isSomeString, typeof(MOD_SYMBOLS))) {
59     auto allTestsWithFunc(string expr, MOD_SYMBOLS...)() pure {
60         //tests is whatever type expr returns
61         ReturnType!(mixin(expr ~ q{!(MOD_SYMBOLS[0])})) tests;
62         foreach(module_; TypeTuple!MOD_SYMBOLS) {
63             tests ~= mixin(expr ~ q{!module_()}); //e.g. tests ~= moduleTestClasses!module_
64         }
65         return tests;
66     }
67 
68     return allTestsWithFunc!(q{moduleTestClasses}, MOD_SYMBOLS) ~
69            allTestsWithFunc!(q{moduleTestFunctions}, MOD_SYMBOLS) ~
70            allTestsWithFunc!(q{moduleUnitTests}, MOD_SYMBOLS);
71 }
72 
73 
74 /**
75  * Finds all built-in unittest blocks in the given module.
76  * @return An array of TestData structs
77  */
78 TestData[] moduleUnitTests(alias module_)() pure nothrow {
79 
80     // Return a name for a unittest block. If no @Name UDA is found a name is
81     // created automatically, else the UDA is used.
82     string unittestName(alias test, int index)() @safe nothrow {
83         import std.conv;
84         mixin("import " ~ fullyQualifiedName!module_ ~ ";"); //so it's visible
85 
86         enum nameAttrs = getUDAs!(test, Name);
87         static assert(nameAttrs.length == 0 || nameAttrs.length == 1, "Found multiple Name UDAs on unittest");
88 
89         enum strAttrs = Filter!(isStringUDA, __traits(getAttributes, test));
90         enum hasName = nameAttrs.length || strAttrs.length == 1;
91         enum prefix = fullyQualifiedName!module_ ~ ".";
92 
93         static if(hasName) {
94             static if(nameAttrs.length == 1)
95                 return prefix ~ nameAttrs[0].value;
96             else
97                 return prefix ~ strAttrs[0];
98         } else {
99             string name;
100             try {
101                 return prefix ~ "unittest" ~ (index).to!string;
102             } catch(Exception) {
103                 assert(false, text("Error converting ", index, " to string"));
104             }
105         }
106     }
107 
108     TestData[] testData;
109     foreach(index, test; __traits(getUnitTests, module_)) {
110         enum name = unittestName!(test, index);
111         enum hidden = hasUDA!(test, HiddenTest);
112         enum shouldFail = hasUDA!(test, ShouldFail);
113         enum singleThreaded = hasUDA!(test, Serial);
114         enum builtin = true;
115 
116         // let's check for @Values UDAs, which are actually of type ValuesImpl
117         enum isValues(alias T) = is(typeof(T)) && is(typeof(T):ValuesImpl!U, U);
118         enum valuesUDAs = Filter!(isValues, __traits(getAttributes, test));
119         static if(valuesUDAs.length == 0) {
120             int i = 1;
121             import unit_threaded.io;
122             testData ~= TestData(name, (){ writelnUt("foo"); test(); }, hidden, shouldFail, singleThreaded, builtin);
123         } else {
124             static assert(valuesUDAs.length == 1, "Can only use @Values once");
125             foreach(i, value; aliasSeqOf!(valuesUDAs[0].values)) {
126                 import std.conv;
127                 // force single threaded so a composite test case is created
128                 // we set a global static to the value the test expects then call the test function,
129                 // which can retrieve the value with getValue!T
130                 auto boo = () {
131                     ValueHolder!(typeof(value)).value = value;
132                     test();
133                 };
134                 testData ~= TestData(name ~ "." ~ value.to!string,
135                                      () {
136                                          ValueHolder!(typeof(value)).value = value;
137                                          test();
138                                      },
139                                      hidden, shouldFail, true /*serial*/, builtin);
140             }
141         }
142     }
143     return testData;
144 }
145 
146 private template isStringUDA(alias T) {
147     static if(__traits(compiles, isSomeString!(typeof(T))))
148         enum isStringUDA = isSomeString!(typeof(T));
149     else
150         enum isStringUDA = false;
151 }
152 
153 unittest {
154     static assert(isStringUDA!"foo");
155     static assert(!isStringUDA!5);
156 }
157 
158 private template isPrivate(alias module_, string moduleMember) {
159     mixin(`import ` ~ fullyQualifiedName!module_ ~ `: ` ~ moduleMember ~ `;`);
160     static if(__traits(compiles, isSomeFunction!(mixin(moduleMember)))) {
161         enum isPrivate = false;
162     } else {
163         enum isPrivate = true;
164     }
165 }
166 
167 
168 // if this member is a test function or class, given the predicate
169 private template PassesTestPred(alias module_, alias pred, string moduleMember) {
170     //should be the line below instead but a compiler bug prevents it
171     //mixin(importMember!module_(moduleMember));
172     mixin("import " ~ fullyQualifiedName!module_ ~ ";");
173     enum notPrivate = __traits(compiles, mixin(moduleMember)); //only way I know to check if private
174     //enum notPrivate = !isPrivate!(module_, moduleMember);
175     static if(notPrivate)
176         enum PassesTestPred = notPrivate && pred!(module_, moduleMember) &&
177                               !HasAttribute!(module_, moduleMember, DontTest);
178     else
179         enum PassesTestPred = false;
180 }
181 
182 
183 /**
184  * Finds all test classes (classes implementing a test() function)
185  * in the given module
186  */
187 TestData[] moduleTestClasses(alias module_)() pure nothrow {
188 
189     template isTestClass(alias module_, string moduleMember) {
190         mixin(importMember!module_(moduleMember));
191         static if(isPrivate!(module_, moduleMember)) {
192             enum isTestClass = false;
193         } else static if(!__traits(compiles, isAggregateType!(mixin(moduleMember)))) {
194             enum isTestClass = false;
195         } else static if(!isAggregateType!(mixin(moduleMember))) {
196             enum isTestClass = false;
197         } else static if(!__traits(compiles, mixin("new " ~ moduleMember))) {
198             enum isTestClass = false; //can't new it, can't use it
199         } else {
200             enum hasUnitTest = HasAttribute!(module_, moduleMember, UnitTest);
201             enum hasTestMethod = __traits(hasMember, mixin(moduleMember), "test");
202             enum isTestClass = hasTestMethod || hasUnitTest;
203         }
204     }
205 
206 
207     return moduleTestData!(module_, isTestClass, memberTestData);
208 }
209 
210 
211 /**
212  * Finds all test functions in the given module.
213  * Returns an array of TestData structs
214  */
215 TestData[] moduleTestFunctions(alias module_)() pure {
216 
217     enum isTypesAttr(alias T) = is(T) && is(T:Types!U, U...);
218 
219     template isTestFunction(alias module_, string moduleMember) {
220         mixin(importMember!module_(moduleMember));
221 
222         static if(isPrivate!(module_, moduleMember)) {
223             enum isTestFunction = false;
224         } else static if(AliasSeq!(mixin(moduleMember)).length != 1) {
225             enum isTestFunction = false;
226         } else static if(isSomeFunction!(mixin(moduleMember))) {
227             enum isTestFunction = hasTestPrefix!(module_, moduleMember) ||
228                                   HasAttribute!(module_, moduleMember, UnitTest);
229         } else {
230             // in this case we handle the possibility of a template function with
231             // the @Types UDA attached to it
232             alias types = GetTypes!(mixin(moduleMember));
233             enum isTestFunction = hasTestPrefix!(module_, moduleMember) &&
234                                   types.length > 0 &&
235                                   is(typeof(() {
236                                       mixin(moduleMember ~ `!` ~ types[0].stringof ~ `;`);
237                                   }));
238         }
239     }
240 
241     template hasTestPrefix(alias module_, string member) {
242         import std.uni: isUpper;
243         mixin(importMember!module_(member));
244 
245         enum prefix = "test";
246         enum minSize = prefix.length + 1;
247 
248         static if(member.length >= minSize && member[0 .. prefix.length] == prefix &&
249                   isUpper(member[prefix.length])) {
250             enum hasTestPrefix = true;
251         } else {
252             enum hasTestPrefix = false;
253         }
254     }
255 
256 
257     return moduleTestData!(module_, isTestFunction, createFuncTestData);
258 }
259 
260 private TestData[] createFuncTestData(alias module_, string moduleMember)() {
261     mixin(importMember!module_(moduleMember));
262     /*
263       Get all the test functions for this module member. There might be more than one
264       when using parametrized unit tests.
265 
266       Examples:
267       ------
268       void testFoo() {} // -> the array contains one element, testFoo
269       @(1, 2, 3) void testBar(int) {} // The array contains 3 elements, one for each UDA value
270       @Types!(int, float) void testBaz(T)() {} //The array contains 2 elements, one for each type
271       ------
272     */
273     // if the predicate returned true (which is always the case here), then it's either
274     // a regular function or a templated one. If regular is has a pointer to it
275     enum isRegularFunction = __traits(compiles, &__traits(getMember, module_, moduleMember));
276 
277     static if(isRegularFunction) {
278 
279         enum func = &__traits(getMember, module_, moduleMember);
280         enum arity = arity!func;
281 
282         static assert(arity == 0 || arity == 1, "Test functions may take at most one parameter");
283 
284         static if(arity == 0)
285             // the reason we're creating a lambda to call the function is that test functions
286             // are ordinary functions, but we're storing delegates
287             return [ memberTestData!(module_, moduleMember)(() { func(); }) ]; //simple case, just call the function
288         else {
289 
290             // the function takes a parameter, check if it has UDAs for value parameters to be passed to it
291             alias params = Parameters!func;
292             static assert(params.length == 1, "Test functions may take at most one parameter");
293 
294             alias values = GetAttributes!(module_, moduleMember, params[0]);
295 
296             import std.conv;
297             static assert(values.length > 0,
298                           text("Test functions with a parameter of type <", params[0].stringof,
299                                "> must have value UDAs of the same type"));
300 
301             TestData[] testData;
302             foreach(v; values)
303                 testData ~= memberTestData!(module_, moduleMember)(() { func(v); }, v.to!string);
304             return testData;
305         }
306     } else static if(HasTypes!(mixin(moduleMember))) { //template function with @Types
307         alias types = GetTypes!(mixin(moduleMember));
308         TestData[] testData;
309         foreach(type; types) {
310             testData ~= memberTestData!(module_, moduleMember)(() {
311                     mixin(moduleMember ~ `!(` ~ type.stringof ~ `)();`);
312                 }, type.stringof);
313         }
314         return testData;
315     } else {
316         return [];
317     }
318 }
319 
320 
321 
322 // this funtion returns TestData for either classes or test functions
323 // built-in unittest modules are handled by moduleUnitTests
324 // pred determines what qualifies as a test
325 // createTestData must return TestData[]
326 private TestData[] moduleTestData(alias module_, alias pred, alias createTestData)() pure {
327     mixin("import " ~ fullyQualifiedName!module_ ~ ";"); //so it's visible
328     TestData[] testData;
329     foreach(moduleMember; __traits(allMembers, module_)) {
330 
331         static if(PassesTestPred!(module_, pred, moduleMember))
332             testData ~= createTestData!(module_, moduleMember);
333     }
334 
335     return testData;
336 
337 }
338 
339 // TestData for a member of a module (either a test function or test class)
340 private TestData memberTestData(alias module_, string moduleMember)(TestFunction testFunction = null, string suffix = "") {
341     //if there is a suffix, all tests sharing that suffix are single threaded with multiple values per "real" test
342     //this is slightly hackish but works and actually makes sense - it causes unit_threaded.factory to make
343     //a CompositeTestCase out of them
344     immutable singleThreaded = HasAttribute!(module_, moduleMember, Serial) || suffix != "";
345     enum builtin = false;
346 
347     return TestData(fullyQualifiedName!module_~ "." ~ moduleMember,
348                     testFunction,
349                     HasAttribute!(module_, moduleMember, HiddenTest),
350                     HasAttribute!(module_, moduleMember, ShouldFail),
351                     singleThreaded,
352                     builtin,
353                     suffix);
354 }
355 
356 version(unittest) {
357 
358     import unit_threaded.tests.module_with_tests; //defines tests and non-tests
359     import unit_threaded.asserts;
360     import std.algorithm;
361     import std.array;
362 
363     //helper function for the unittest blocks below
364     private auto addModPrefix(string[] elements,
365                               string module_ = "unit_threaded.tests.module_with_tests") nothrow {
366         return elements.map!(a => module_ ~ "." ~ a).array;
367     }
368 }
369 
370 unittest {
371     const expected = addModPrefix([ "FooTest", "BarTest", "Blergh"]);
372     const actual = moduleTestClasses!(unit_threaded.tests.module_with_tests).
373         map!(a => a.name).array;
374     assertEqual(actual, expected);
375 }
376 
377 unittest {
378     const expected = addModPrefix([ "testFoo", "testBar", "funcThatShouldShowUpCosOfAttr"]);
379     const actual = moduleTestFunctions!(unit_threaded.tests.module_with_tests).
380         map!(a => a.getPath).array;
381     assertEqual(actual, expected);
382 }
383 
384 
385 unittest {
386     const expected = addModPrefix(["unittest0", "unittest1", "myUnitTest"]);
387     const actual = moduleUnitTests!(unit_threaded.tests.module_with_tests).
388         map!(a => a.name).array;
389     assertEqual(actual, expected);
390 }
391 
392 version(unittest) {
393     import unit_threaded.testcase: TestCase;
394     private void assertFail(TestCase test, string file = __FILE__, ulong line = __LINE__) {
395         import core.exception;
396         import std.conv;
397 
398         try {
399             test.silence;
400             assert(test() != [],
401                    file ~ ":" ~ line.to!string ~ " Test was expected to fail but didn't");
402             assert(false, file ~ ":" ~ line.to!string ~ " Expected test case " ~ test.getPath ~
403                    " to fail with AssertError but it didn't");
404         } catch(AssertError) {}
405     }
406 }
407 
408 @("Test that parametrized value tests work")
409 unittest {
410     import unit_threaded.factory;
411     import unit_threaded.testcase;
412 
413     const testData = allTestData!(unit_threaded.tests.parametrized).
414         filter!(a => a.name.endsWith("testValues")).array;
415 
416     // there should only be on test case which is a composite of the 3 values in testValues
417     auto composite = cast(CompositeTestCase)createTestCases(testData)[0];
418     assert(composite !is null, "Wrong dynamic type for TestCase");
419     auto tests = composite.tests;
420     assertEqual(tests.length, 3);
421 
422     // the first and third test should pass, the second should fail
423     assertEqual(tests[0](), []);
424     assertEqual(tests[2](), []);
425 
426     assertFail(tests[1]);
427 }
428 
429 
430 @("Test that parametrized type tests work")
431 unittest {
432     import unit_threaded.factory;
433     import unit_threaded.testcase;
434 
435     const testData = allTestData!(unit_threaded.tests.parametrized).
436         filter!(a => a.name.endsWith("testTypes")).array;
437     const expected = addModPrefix(["testTypes.float", "testTypes.int"],
438                                   "unit_threaded.tests.parametrized");
439     const actual = testData.map!(a => a.getPath).array;
440     assertEqual(actual, expected);
441 
442     // there should only be on test case which is a composite of the 2 testTypes
443     auto composite = cast(CompositeTestCase)createTestCases(testData)[0];
444     assert(composite !is null, "Wrong dynamic type for TestCase");
445     auto tests = composite.tests;
446     assertEqual(tests.map!(a => a.getPath).array, expected);
447 
448     // the second should pass, the first should fail
449     assertEqual(tests[1](), []);
450 
451     assertFail(tests[0]);
452 }
453 
454 @("Test that int value parametrized built-in unittest blocks work")
455 unittest {
456     import unit_threaded.factory;
457     import unit_threaded.testcase;
458 
459     const testData = allTestData!(unit_threaded.tests.parametrized).
460         filter!(a => a.name.canFind("builtinIntValues")).array;
461 
462     // there should only be on test case which is a composite of the 4 values
463     auto composite = cast(CompositeTestCase)createTestCases(testData)[0];
464     assert(composite !is null, "Wrong dynamic type for TestCase");
465     auto tests = composite.tests;
466     assertEqual(tests.length, 4);
467 
468     assertEqual(tests[1](), []);
469 
470     //these should fail
471     assertFail(tests[0]);
472     assertFail(tests[2]);
473     assertFail(tests[3]);
474 }