1 module unit_threaded.reflection;
2 
3 import unit_threaded.attrs;
4 import unit_threaded.uda;
5 import unit_threaded.meta;
6 import std.traits;
7 import std.meta;
8 
9 /**
10  * Common data for test functions and test classes
11  */
12 alias void delegate() TestFunction;
13 struct TestData {
14     string name;
15     TestFunction testFunction; ///only used for functions, null for classes
16     bool hidden;
17     bool shouldFail;
18     bool singleThreaded;
19     bool builtin;
20     string suffix; // append to end of getPath
21     string[] tags;
22 
23     string getPath() const pure nothrow {
24         string path = name.dup;
25         import std.array: empty;
26         if(!suffix.empty) path ~= "." ~ suffix;
27         return path;
28     }
29 
30     bool isTestClass() @safe const pure nothrow {
31         return testFunction is null;
32     }
33 }
34 
35 
36 /**
37  * Finds all test cases (functions, classes, built-in unittest blocks)
38  * Template parameters are module strings
39  */
40 const(TestData)[] allTestData(MOD_STRINGS...)() if(allSatisfy!(isSomeString, typeof(MOD_STRINGS))) {
41 
42     string getModulesString() {
43         import std.array: join;
44         string[] modules;
45         foreach(module_; MOD_STRINGS) modules ~= module_;
46         return modules.join(", ");
47     }
48 
49     enum modulesString =  getModulesString;
50     mixin("import " ~ modulesString ~ ";");
51     mixin("return allTestData!(" ~ modulesString ~ ");");
52 }
53 
54 
55 /**
56  * Finds all test cases (functions, classes, built-in unittest blocks)
57  * Template parameters are module symbols
58  */
59 const(TestData)[] allTestData(MOD_SYMBOLS...)() if(!anySatisfy!(isSomeString, typeof(MOD_SYMBOLS))) {
60     auto allTestsWithFunc(string expr, MOD_SYMBOLS...)() pure {
61         //tests is whatever type expr returns
62         ReturnType!(mixin(expr ~ q{!(MOD_SYMBOLS[0])})) tests;
63         foreach(module_; AliasSeq!MOD_SYMBOLS) {
64             tests ~= mixin(expr ~ q{!module_()}); //e.g. tests ~= moduleTestClasses!module_
65         }
66         return tests;
67     }
68 
69     return allTestsWithFunc!(q{moduleTestClasses}, MOD_SYMBOLS) ~
70            allTestsWithFunc!(q{moduleTestFunctions}, MOD_SYMBOLS) ~
71            allTestsWithFunc!(q{moduleUnitTests}, MOD_SYMBOLS);
72 }
73 
74 
75 /**
76  * Finds all built-in unittest blocks in the given module.
77  * Recurses into structs, classes, and unions of the module.
78  *
79  * @return An array of TestData structs
80  */
81 TestData[] moduleUnitTests(alias module_)() pure nothrow {
82 
83     // Return a name for a unittest block. If no @Name UDA is found a name is
84     // created automatically, else the UDA is used.
85     // the weird name for the first template parameter is so that it doesn't clash
86     // with a package name
87     string unittestName(alias _theUnitTest, int index)() @safe nothrow {
88         import std.conv;
89         mixin("import " ~ fullyQualifiedName!module_ ~ ";"); //so it's visible
90 
91         enum nameAttrs = getUDAs!(_theUnitTest, Name);
92         static assert(nameAttrs.length == 0 || nameAttrs.length == 1, "Found multiple Name UDAs on unittest");
93 
94         enum strAttrs = Filter!(isStringUDA, __traits(getAttributes, _theUnitTest));
95         enum hasName = nameAttrs.length || strAttrs.length == 1;
96         enum prefix = fullyQualifiedName!(__traits(parent, _theUnitTest)) ~ ".";
97 
98         static if(hasName) {
99             static if(nameAttrs.length == 1)
100                 return prefix ~ nameAttrs[0].value;
101             else
102                 return prefix ~ strAttrs[0];
103         } else {
104             string name;
105             try {
106                 return prefix ~ "unittest" ~ (index).to!string;
107             } catch(Exception) {
108                 assert(false, text("Error converting ", index, " to string"));
109             }
110         }
111     }
112 
113 
114     TestData[] testData;
115 
116     void addMemberUnittests(alias member)() pure nothrow{
117         foreach(index, eltesto; __traits(getUnitTests, member)) {
118             enum name = unittestName!(eltesto, index);
119             enum hidden = hasUDA!(eltesto, HiddenTest);
120             enum shouldFail = hasUDA!(eltesto, ShouldFail);
121             enum singleThreaded = hasUDA!(eltesto, Serial);
122             enum builtin = true;
123             enum suffix = "";
124 
125             // let's check for @Values UDAs, which are actually of type ValuesImpl
126             enum isValues(alias T) = is(typeof(T)) && is(typeof(T):ValuesImpl!U, U);
127             alias valuesUDAs = Filter!(isValues, __traits(getAttributes, eltesto));
128 
129             enum isTags(alias T) = is(typeof(T)) && is(typeof(T) == Tags);
130             enum tags = tagsFromAttrs!(Filter!(isTags, __traits(getAttributes, eltesto)));
131 
132             static if(valuesUDAs.length == 0) {
133                 testData ~= TestData(name, (){ eltesto(); }, hidden, shouldFail, singleThreaded, builtin, suffix, tags);
134             } else {
135                 import std.range;
136 
137                 // cartesianProduct doesn't work with only one range, so in the usual case
138                 // of only one @Values UDA, we bind to prod with a range of tuples, just
139                 // as returned by cartesianProduct.
140 
141                 static if(valuesUDAs.length == 1) {
142                     import std.typecons;
143                     enum prod = valuesUDAs[0].values.map!(a => tuple(a));
144                 } else {
145                     mixin(`enum prod = cartesianProduct(` ~ valuesUDAs.length.iota.map!
146                           (a => `valuesUDAs[` ~ guaranteedToString(a) ~ `].values`).join(", ") ~ `);`);
147                 }
148 
149                 foreach(comb; aliasSeqOf!prod) {
150                     enum valuesName = valuesName(comb);
151 
152                     static if(hasUDA!(eltesto, AutoTags))
153                         enum realTags = tags ~ valuesName.split(".").array;
154                     else
155                         enum realTags = tags;
156 
157                     testData ~= TestData(name ~ "." ~ valuesName,
158                                          () {
159                                              foreach(i; aliasSeqOf!(comb.length.iota))
160                                                  ValueHolder!(typeof(comb[i])).values[i] = comb[i];
161                                              eltesto();
162                                          },
163                                          hidden, shouldFail, singleThreaded, builtin, suffix, realTags);
164 
165                 }
166             }
167         }
168     }
169 
170 
171     // Keeps track of mangled names of everything visited.
172     bool[string] visitedMembers;
173 
174     void addUnitTestsRecursively(alias composite)() pure nothrow {
175         mixin("import " ~ fullyQualifiedName!module_ ~ ";"); //so it's visible
176 
177         if (composite.mangleof in visitedMembers)
178             return;
179         visitedMembers[composite.mangleof] = true;
180         addMemberUnittests!composite();
181         foreach(member; __traits(allMembers, composite)){
182             enum notPrivate = __traits(compiles, mixin(member)); //only way I know to check if private
183             static if (
184                 notPrivate &&
185                 // If visibility of the member is deprecated, the next line still returns true
186                 // and yet spills deprecation warning. If deprecation is turned into error,
187                 // all works as intended.
188                 __traits(compiles, __traits(getMember, composite, member)) &&
189                 __traits(compiles, __traits(allMembers, __traits(getMember, composite, member))) &&
190                 __traits(compiles, recurse!(__traits(getMember, composite, member)))
191             ) {
192                 recurse!(__traits(getMember, composite, member));
193             }
194         }
195     }
196 
197     void recurse(child)() pure nothrow {
198         enum notPrivate = __traits(compiles, child.init); //only way I know to check if private
199         static if (is(child == class) || is(child == struct) || is(child == union)) {
200             addUnitTestsRecursively!child;
201         }
202     }
203 
204     addUnitTestsRecursively!module_();
205     return testData;
206 }
207 
208 private string valuesName(T)(T tuple) {
209     import std.algorithm;
210     import std.range;
211     string[] parts;
212     foreach(a; aliasSeqOf!(tuple.length.iota))
213         parts ~= guaranteedToString(tuple[a]);
214     return parts.join(".");
215 }
216 
217 private string guaranteedToString(T)(T value) nothrow pure @safe {
218     import std.conv;
219     try
220         return value.to!string;
221     catch(Exception ex)
222         assert(0, "Could not convert value to string");
223 }
224 
225 private string getValueAsString(T)(T value) nothrow pure @safe {
226     import std.conv;
227     try
228         return value.to!string;
229     catch(Exception ex)
230         assert(0, "Could not convert value to string");
231 }
232 
233 
234 private template isStringUDA(alias T) {
235     static if(__traits(compiles, isSomeString!(typeof(T))))
236         enum isStringUDA = isSomeString!(typeof(T));
237     else
238         enum isStringUDA = false;
239 }
240 
241 unittest {
242     static assert(isStringUDA!"foo");
243     static assert(!isStringUDA!5);
244 }
245 
246 private template isPrivate(alias module_, string moduleMember) {
247     mixin(`import ` ~ fullyQualifiedName!module_ ~ `: ` ~ moduleMember ~ `;`);
248     static if(__traits(compiles, isSomeFunction!(mixin(moduleMember)))) {
249         enum isPrivate = false;
250     } else {
251         enum isPrivate = true;
252     }
253 }
254 
255 
256 // if this member is a test function or class, given the predicate
257 private template PassesTestPred(alias module_, alias pred, string moduleMember) {
258     //should be the line below instead but a compiler bug prevents it
259     //mixin(importMember!module_(moduleMember));
260     mixin("import " ~ fullyQualifiedName!module_ ~ ";");
261     alias I(T...) = T;
262     alias member = I!(__traits(getMember, module_, moduleMember));
263 
264     template canCheckIfSomeFunction(T...) {
265         enum canCheckIfSomeFunction = T.length == 1 && __traits(compiles, isSomeFunction!(T[0]));
266     }
267 
268     private string funcCallMixin(alias T)() {
269         import std.conv: to;
270         string[] args;
271         foreach(i, ParamType; Parameters!T) {
272             args ~= `arg` ~ i.to!string;
273         }
274 
275         return moduleMember ~ `(` ~ args.join(`,`) ~ `);`;
276     }
277 
278     private string argsMixin(alias T)() {
279         import std.conv: to;
280         string[] args;
281         foreach(i, ParamType; Parameters!T) {
282             args ~= ParamType.stringof ~ ` arg` ~ i.to!string ~ `;`;
283         }
284 
285         return args.join("\n");
286     }
287 
288     template canCallMember() {
289         void _f() {
290             mixin(argsMixin!member);
291             mixin(funcCallMixin!member);
292         }
293     }
294 
295     template canInstantiate() {
296         void _f() {
297             mixin(`auto _ = new ` ~ moduleMember ~ `;`);
298         }
299     }
300 
301     template isPrivate() {
302         static if(!canCheckIfSomeFunction!member) {
303             enum isPrivate = !__traits(compiles, __traits(getMember, module_, moduleMember));
304         } else {
305             static if(isSomeFunction!member) {
306                 enum isPrivate = !__traits(compiles, canCallMember!());
307             } else static if(is(member)) {
308                 static if(isAggregateType!member)
309                     enum isPrivate = !__traits(compiles, canInstantiate!());
310                 else
311                     enum isPrivate = !__traits(compiles, __traits(getMember, module_, moduleMember));
312             } else {
313                 enum isPrivate = !__traits(compiles, __traits(getMember, module_, moduleMember));
314             }
315         }
316     }
317 
318     enum notPrivate = !isPrivate!();
319     enum PassesTestPred = !isPrivate!() && pred!(module_, moduleMember) &&
320         !HasAttribute!(module_, moduleMember, DontTest);
321 }
322 
323 
324 /**
325  * Finds all test classes (classes implementing a test() function)
326  * in the given module
327  */
328 TestData[] moduleTestClasses(alias module_)() pure nothrow {
329 
330     template isTestClass(alias module_, string moduleMember) {
331         mixin(importMember!module_(moduleMember));
332         static if(isPrivate!(module_, moduleMember)) {
333             enum isTestClass = false;
334         } else static if(!__traits(compiles, isAggregateType!(mixin(moduleMember)))) {
335             enum isTestClass = false;
336         } else static if(!isAggregateType!(mixin(moduleMember))) {
337             enum isTestClass = false;
338         } else static if(!__traits(compiles, mixin("new " ~ moduleMember))) {
339             enum isTestClass = false; //can't new it, can't use it
340         } else {
341             enum hasUnitTest = HasAttribute!(module_, moduleMember, UnitTest);
342             enum hasTestMethod = __traits(hasMember, mixin(moduleMember), "test");
343             enum isTestClass = hasTestMethod || hasUnitTest;
344         }
345     }
346 
347 
348     return moduleTestData!(module_, isTestClass, memberTestData);
349 }
350 
351 
352 /**
353  * Finds all test functions in the given module.
354  * Returns an array of TestData structs
355  */
356 TestData[] moduleTestFunctions(alias module_)() pure {
357 
358     enum isTypesAttr(alias T) = is(T) && is(T:Types!U, U...);
359 
360     template isTestFunction(alias module_, string moduleMember) {
361         mixin(importMember!module_(moduleMember));
362 
363         static if(isPrivate!(module_, moduleMember)) {
364             enum isTestFunction = false;
365         } else static if(AliasSeq!(mixin(moduleMember)).length != 1) {
366             enum isTestFunction = false;
367         } else static if(isSomeFunction!(mixin(moduleMember))) {
368             enum isTestFunction = hasTestPrefix!(module_, moduleMember) ||
369                                   HasAttribute!(module_, moduleMember, UnitTest);
370         } else static if(__traits(compiles, __traits(getAttributes, mixin(moduleMember)))) {
371             // in this case we handle the possibility of a template function with
372             // the @Types UDA attached to it
373             alias types = GetTypes!(mixin(moduleMember));
374             enum isTestFunction = hasTestPrefix!(module_, moduleMember) &&
375                                   types.length > 0;
376         } else {
377             enum isTestFunction = false;
378         }
379 
380     }
381 
382     template hasTestPrefix(alias module_, string member) {
383         import std.uni: isUpper;
384         mixin(importMember!module_(member));
385 
386         enum prefix = "test";
387         enum minSize = prefix.length + 1;
388 
389         static if(member.length >= minSize && member[0 .. prefix.length] == prefix &&
390                   isUpper(member[prefix.length])) {
391             enum hasTestPrefix = true;
392         } else {
393             enum hasTestPrefix = false;
394         }
395     }
396 
397     return moduleTestData!(module_, isTestFunction, createFuncTestData);
398 }
399 
400 private TestData[] createFuncTestData(alias module_, string moduleMember)() {
401     mixin(importMember!module_(moduleMember));
402     /*
403       Get all the test functions for this module member. There might be more than one
404       when using parametrized unit tests.
405 
406       Examples:
407       ------
408       void testFoo() {} // -> the array contains one element, testFoo
409       @(1, 2, 3) void testBar(int) {} // The array contains 3 elements, one for each UDA value
410       @Types!(int, float) void testBaz(T)() {} //The array contains 2 elements, one for each type
411       ------
412     */
413     // if the predicate returned true (which is always the case here), then it's either
414     // a regular function or a templated one. If regular we can get a pointer to it
415     enum isRegularFunction = __traits(compiles, &__traits(getMember, module_, moduleMember));
416 
417     static if(isRegularFunction) {
418 
419         enum func = &__traits(getMember, module_, moduleMember);
420         enum arity = arity!func;
421 
422         static if(arity == 0)
423             // the reason we're creating a lambda to call the function is that test functions
424             // are ordinary functions, but we're storing delegates
425             return [ memberTestData!(module_, moduleMember)(() { func(); }) ]; //simple case, just call the function
426         else {
427 
428             // the function has parameters, check if it has UDAs for value parameters to be passed to it
429             alias params = Parameters!func;
430 
431             import std.range: iota;
432             import std.algorithm: any;
433             import std.typecons: tuple, Tuple;
434 
435             bool hasAttributesForAllParams() {
436                 auto ret = true;
437                 foreach(p; params) {
438                     if(tuple(GetAttributes!(module_, moduleMember, p)).length == 0) {
439                         ret = false;
440                     }
441                 }
442                 return ret;
443             }
444 
445             static if(!hasAttributesForAllParams) {
446                 import std.conv: text;
447                 pragma(msg, text("Warning: ", moduleMember, " passes the criteria for a value-parameterized test function",
448                                  " but doesn't have the appropriate value UDAs.\n",
449                                  "         Consider changing its name or annotating it with @DontTest"));
450                 return [];
451             } else {
452 
453                 static if(arity == 1) {
454                     // bind a range of tuples to prod just as cartesianProduct returns
455                     enum prod = [GetAttributes!(module_, moduleMember, params[0])].map!(a => tuple(a));
456                 } else {
457                     import std.conv: text;
458 
459                     mixin(`enum prod = cartesianProduct(` ~ params.length.iota.map!
460                           (a => `[GetAttributes!(module_, moduleMember, params[` ~ guaranteedToString(a) ~ `])]`).join(", ") ~ `);`);
461                 }
462 
463                 TestData[] testData;
464                 foreach(comb; aliasSeqOf!prod) {
465                     enum valuesName = valuesName(comb);
466 
467                     static if(HasAttribute!(module_, moduleMember, AutoTags))
468                         enum extraTags = valuesName.split(".").array;
469                     else
470                         enum string[] extraTags = [];
471 
472 
473                     testData ~= memberTestData!(module_, moduleMember, extraTags)(
474                         // func(value0, value1, ...)
475                         () { func(comb.expand); },
476                         valuesName);
477                 }
478 
479                 return testData;
480             }
481         }
482     } else static if(HasTypes!(mixin(moduleMember))) { //template function with @Types
483         alias types = GetTypes!(mixin(moduleMember));
484         TestData[] testData;
485         foreach(type; types) {
486 
487             static if(HasAttribute!(module_, moduleMember, AutoTags))
488                 enum extraTags = [type.stringof];
489             else
490                 enum string[] extraTags = [];
491 
492             static if(!__traits(compiles, mixin(moduleMember ~ `!(` ~ type.stringof ~ `)()`))) {
493                 pragma(msg, "Could not compile Type-parameterized test for T = ", type);
494                 void _failFunc() {
495                     mixin(moduleMember ~ `!(` ~ type.stringof ~ `)();`);
496                 }
497                 static assert(false);
498             }
499 
500             testData ~= memberTestData!(module_, moduleMember, extraTags)(
501                 () {
502                     mixin(moduleMember ~ `!(` ~ type.stringof ~ `)();`);
503                 },
504                 type.stringof);
505         }
506         return testData;
507     } else {
508         return [];
509     }
510 }
511 
512 
513 
514 // this funtion returns TestData for either classes or test functions
515 // built-in unittest modules are handled by moduleUnitTests
516 // pred determines what qualifies as a test
517 // createTestData must return TestData[]
518 private TestData[] moduleTestData(alias module_, alias pred, alias createTestData)() pure {
519     mixin("import " ~ fullyQualifiedName!module_ ~ ";"); //so it's visible
520     TestData[] testData;
521     foreach(moduleMember; __traits(allMembers, module_)) {
522 
523         static if(PassesTestPred!(module_, pred, moduleMember))
524             testData ~= createTestData!(module_, moduleMember);
525     }
526 
527     return testData;
528 
529 }
530 
531 // TestData for a member of a module (either a test function or test class)
532 private TestData memberTestData(alias module_, string moduleMember, string[] extraTags = [])
533     (TestFunction testFunction = null, string suffix = "") {
534     immutable singleThreaded = HasAttribute!(module_, moduleMember, Serial);
535     enum builtin = false;
536     enum tags = tagsFromAttrs!(GetAttributes!(module_, moduleMember, Tags));
537 
538     return TestData(fullyQualifiedName!module_~ "." ~ moduleMember,
539                     testFunction,
540                     HasAttribute!(module_, moduleMember, HiddenTest),
541                     HasAttribute!(module_, moduleMember, ShouldFail),
542                     singleThreaded,
543                     builtin,
544                     suffix,
545                     tags ~ extraTags);
546 }
547 
548 string[] tagsFromAttrs(T...)() {
549     static assert(T.length <= 1, "@Tags can only be applied once");
550     static if(T.length)
551         return T[0].values;
552     else
553         return [];
554 }
555 
556 version(unittest) {
557 
558     import unit_threaded.tests.module_with_tests; //defines tests and non-tests
559     import unit_threaded.asserts;
560     import std.algorithm;
561     import std.array;
562 
563     //helper function for the unittest blocks below
564     private auto addModPrefix(string[] elements,
565                               string module_ = "unit_threaded.tests.module_with_tests") nothrow {
566         return elements.map!(a => module_ ~ "." ~ a).array;
567     }
568 }
569 
570 unittest {
571     const expected = addModPrefix([ "FooTest", "BarTest", "Blergh"]);
572     const actual = moduleTestClasses!(unit_threaded.tests.module_with_tests).
573         map!(a => a.name).array;
574     assertEqual(actual, expected);
575 }
576 
577 unittest {
578     const expected = addModPrefix([ "testFoo", "testBar", "funcThatShouldShowUpCosOfAttr"]);
579     const actual = moduleTestFunctions!(unit_threaded.tests.module_with_tests).
580         map!(a => a.getPath).array;
581     assertEqual(actual, expected);
582 }
583 
584 
585 unittest {
586     const expected = addModPrefix(["unittest0", "unittest1", "myUnitTest",
587                                    "StructWithUnitTests.InStruct", "StructWithUnitTests.unittest1"]);
588     const actual = moduleUnitTests!(unit_threaded.tests.module_with_tests).
589         map!(a => a.name).array;
590     assertEqual(actual, expected);
591 }
592 
593 version(unittest) {
594     import unit_threaded.testcase: TestCase;
595     private void assertFail(TestCase test, string file = __FILE__, size_t line = __LINE__) {
596         import core.exception;
597         import std.conv;
598 
599         try {
600             test.silence;
601             assert(test() != [], file ~ ":" ~ line.to!string ~ " Test was expected to fail but didn't");
602             assert(false, file ~ ":" ~ line.to!string ~ " Expected test case " ~ test.getPath ~
603                    " to fail with AssertError but it didn't");
604         } catch(AssertError) {}
605     }
606 
607     private void assertPass(TestCase test, string file = __FILE__, size_t line = __LINE__) {
608         assertEqual(test(), [], file, line);
609     }
610 }
611 
612 @("Test that parametrized value tests work")
613 unittest {
614     import unit_threaded.factory;
615     import unit_threaded.testcase;
616     import unit_threaded.tests.parametrized;
617 
618     const testData = allTestData!(unit_threaded.tests.parametrized).
619         filter!(a => a.name.endsWith("testValues")).array;
620 
621     auto tests = createTestCases(testData);
622     assertEqual(tests.length, 3);
623 
624     // the first and third test should pass, the second should fail
625     assertPass(tests[0]);
626     assertPass(tests[2]);
627 
628     assertFail(tests[1]);
629 }
630 
631 
632 @("Test that parametrized type tests work")
633 unittest {
634     import unit_threaded.factory;
635     import unit_threaded.testcase;
636     import unit_threaded.tests.parametrized;
637 
638     const testData = allTestData!(unit_threaded.tests.parametrized).
639         filter!(a => a.name.endsWith("testTypes")).array;
640     const expected = addModPrefix(["testTypes.float", "testTypes.int"],
641                                   "unit_threaded.tests.parametrized");
642     const actual = testData.map!(a => a.getPath).array;
643     assertEqual(actual, expected);
644 
645     auto tests = createTestCases(testData);
646     assertEqual(tests.map!(a => a.getPath).array, expected);
647 
648     assertPass(tests[1]);
649     assertFail(tests[0]);
650 }
651 
652 @("Value parametrized built-in unittests")
653 unittest {
654     import unit_threaded.factory;
655     import unit_threaded.testcase;
656     import unit_threaded.tests.parametrized;
657 
658     const testData = allTestData!(unit_threaded.tests.parametrized).
659         filter!(a => a.name.canFind("builtinIntValues")).array;
660 
661     auto tests = createTestCases(testData);
662     assertEqual(tests.length, 4);
663 
664     // these should be ok
665     assertPass(tests[1]);
666 
667     //these should fail
668     assertFail(tests[0]);
669     assertFail(tests[2]);
670     assertFail(tests[3]);
671 }
672 
673 
674 @("Tests can be selected by tags") unittest {
675     import unit_threaded.factory;
676     import unit_threaded.testcase;
677     import unit_threaded.tests.tags;
678 
679     const testData = allTestData!(unit_threaded.tests.tags).array;
680     auto testsNoTags = createTestCases(testData);
681     assertEqual(testsNoTags.length, 4);
682     assertPass(testsNoTags[0]);
683     assertFail(testsNoTags[1]);
684     assertFail(testsNoTags[2]);
685     assertFail(testsNoTags[3]);
686 
687     auto testsNinja = createTestCases(testData, ["@ninja"]);
688     assertEqual(testsNinja.length, 1);
689     assertPass(testsNinja[0]);
690 
691     auto testsMake = createTestCases(testData, ["@make"]);
692     assertEqual(testsMake.length, 3);
693     assertPass(testsMake.find!(a => a.getPath.canFind("testMake")).front);
694     assertPass(testsMake.find!(a => a.getPath.canFind("unittest0")).front);
695     assertFail(testsMake.find!(a => a.getPath.canFind("unittest2")).front);
696 
697     auto testsNotNinja = createTestCases(testData, ["~@ninja"]);
698     assertEqual(testsNotNinja.length, 3);
699     assertPass(testsNotNinja.find!(a => a.getPath.canFind("testMake")).front);
700     assertFail(testsNotNinja.find!(a => a.getPath.canFind("unittest1")).front);
701     assertFail(testsNotNinja.find!(a => a.getPath.canFind("unittest2")).front);
702 
703     assertEqual(createTestCases(testData, ["unit_threaded.tests.tags.testMake", "@ninja"]).length, 0);
704 }
705 
706 @("Parametrized built-in tests with @AutoTags get tagged by value")
707 unittest {
708     import unit_threaded.factory;
709     import unit_threaded.testcase;
710     import unit_threaded.tests.parametrized;
711 
712     const testData = allTestData!(unit_threaded.tests.parametrized).
713         filter!(a => a.name.canFind("builtinIntValues")).array;
714 
715     auto two = createTestCases(testData, ["@2"]);
716 
717     assertEqual(two.length, 1);
718     assertFail(two[0]);
719 
720     auto three = createTestCases(testData, ["@3"]);
721     assertEqual(three.length, 1);
722     assertPass(three[0]);
723 }
724 
725 @("Value parametrized function tests with @AutoTags get tagged by value")
726 unittest {
727     import unit_threaded.factory;
728     import unit_threaded.testcase;
729     import unit_threaded.tests.parametrized;
730 
731     const testData = allTestData!(unit_threaded.tests.parametrized).
732         filter!(a => a.name.canFind("testValues")).array;
733 
734     auto two = createTestCases(testData, ["@2"]);
735     assertEqual(two.length, 1);
736     assertFail(two[0]);
737 }
738 
739 @("Type parameterized tests with @AutoTags get tagged by type")
740 unittest {
741     import unit_threaded.factory;
742     import unit_threaded.testcase;
743     import unit_threaded.tests.parametrized;
744 
745     const testData = allTestData!(unit_threaded.tests.parametrized).
746         filter!(a => a.name.canFind("testTypes")).array;
747 
748     auto tests = createTestCases(testData, ["@int"]);
749     assertEqual(tests.length, 1);
750     assertPass(tests[0]);
751 }
752 
753 @("Cartesian parameterized built-in values") unittest {
754     import unit_threaded.factory;
755     import unit_threaded.testcase;
756     import unit_threaded.should;
757     import unit_threaded.tests.parametrized;
758 
759     const testData = allTestData!(unit_threaded.tests.parametrized).
760         filter!(a => a.name.canFind("cartesianBuiltinNoAutoTags")).array;
761 
762     auto tests = createTestCases(testData);
763     tests.map!(a => a.getPath).array.shouldBeSameSetAs(
764                 addModPrefix(["foo.red", "foo.blue", "foo.green", "bar.red", "bar.blue", "bar.green"].
765                              map!(a => "cartesianBuiltinNoAutoTags." ~ a).array,
766                              "unit_threaded.tests.parametrized"));
767     assertEqual(tests.length, 6);
768 
769     auto fooRed = tests.find!(a => a.getPath.canFind("foo.red")).front;
770     assertPass(fooRed);
771     assertEqual(getValue!(string, 0), "foo");
772     assertEqual(getValue!(string, 1), "red");
773     assertEqual(testData.find!(a => a.getPath.canFind("foo.red")).front.tags, []);
774 
775     auto barGreen = tests.find!(a => a.getPath.canFind("bar.green")).front;
776     assertFail(barGreen);
777     assertEqual(getValue!(string, 0), "bar");
778     assertEqual(getValue!(string, 1), "green");
779 
780     assertEqual(testData.find!(a => a.getPath.canFind("bar.green")).front.tags, []);
781     assertEqual(allTestData!(unit_threaded.tests.parametrized).
782                 filter!(a => a.name.canFind("cartesianBuiltinAutoTags")).array.
783                 find!(a => a.getPath.canFind("bar.green")).front.tags,
784                 ["bar", "green"]);
785 }
786 
787 @("Cartesian parameterized function values") unittest {
788     import unit_threaded.factory;
789     import unit_threaded.testcase;
790     import unit_threaded.should;
791 
792     const testData = allTestData!(unit_threaded.tests.parametrized).
793         filter!(a => a.name.canFind("CartesianFunction")).array;
794 
795     auto tests = createTestCases(testData);
796         tests.map!(a => a.getPath).array.shouldBeSameSetAs(
797             addModPrefix(["1.foo", "1.bar", "2.foo", "2.bar", "3.foo", "3.bar"].
798                              map!(a => "testCartesianFunction." ~ a).array,
799                              "unit_threaded.tests.parametrized"));
800 
801     foreach(test; tests) {
802         test.getPath.canFind("2.bar")
803             ? assertPass(test)
804             : assertFail(test);
805     }
806 
807     assertEqual(testData.find!(a => a.getPath.canFind("2.bar")).front.tags,
808                 ["2", "bar"]);
809 
810 }
811 
812 unittest {
813     import unit_threaded.testcase;
814     import unit_threaded.factory;
815     import unit_threaded.randomized.gen;
816 
817     auto testData = allTestData!(unit_threaded.randomized.gen).array;
818     auto tests = createTestCases(testData);
819     foreach(test; tests) {
820 
821     }
822 }
823 
824 @("issue 33") unittest {
825     import unit_threaded.factory;
826     import unit_threaded.testcase;
827     import unit_threaded.should;
828     import test.issue33;
829 
830     const testData = allTestData!"test.issue33";
831     assertEqual(testData.length, 1);
832 }
833 
834 @("issue 43") unittest {
835     import unit_threaded.factory;
836     import unit_threaded.asserts;
837     import unit_threaded.tests.module_with_tests;
838     import std.algorithm: canFind;
839     import std.array: array;
840 
841     const testData = allTestData!"unit_threaded.tests.module_with_tests";
842     assertEqual(testData.canFind!(a => a.getPath.canFind("InStruct" )), true);
843     auto inStructTest = testData
844         .find!(a => a.getPath.canFind("InStruct"))
845         .array
846         .createTestCases[0];
847     assertFail(inStructTest);
848 }