1 module unit_threaded.randomized.benchmark;
2 
3 import core.time : MonoTimeImpl, Duration, ClockType, dur, seconds;
4 import std.array : appender, array;
5 import std.datetime : Clock;
6 import std.traits : fullyQualifiedName, Parameters, ParameterIdentifierTuple;
7 
8 import unit_threaded;
9 import unit_threaded.randomized.gen;
10 
11 /* This function used $(D MonoTimeImpl!(ClockType.precise).currTime) to time
12 how long $(D MonoTimeImpl!(ClockType.precise).currTime) takes to return
13 the current time.
14 */
15 private auto medianStopWatchTime()
16 {
17     import core.time;
18     import std.algorithm : sort;
19 
20     enum numRounds = 51;
21     Duration[numRounds] times;
22 
23     MonoTimeImpl!(ClockType.precise) dummy;
24     for (size_t i = 0; i < numRounds; ++i)
25     {
26         auto sw = MonoTimeImpl!(ClockType.precise).currTime;
27         dummy = MonoTimeImpl!(ClockType.precise).currTime;
28         dummy = MonoTimeImpl!(ClockType.precise).currTime;
29         doNotOptimizeAway(dummy);
30         times[i] = MonoTimeImpl!(ClockType.precise).currTime - sw;
31     }
32 
33     sort(times[]);
34 
35     return times[$ / 2].total!"hnsecs";
36 }
37 
38 private Duration getQuantilTick(double q)(Duration[] ticks) pure @safe
39 {
40     size_t idx = cast(size_t)(ticks.length * q);
41 
42     if (ticks.length % 2 == 1)
43     {
44         return ticks[idx];
45     }
46     else
47     {
48         return (ticks[idx] + ticks[idx - 1]) / 2;
49     }
50 }
51 
52 // @Name("Quantil calculations")
53 // unittest
54 // {
55 //     static import std.conv;
56 //     import std.algorithm.iteration : map;
57 
58 //     auto ticks = [1, 2, 3, 4, 5].map!(a => dur!"seconds"(a)).array;
59 
60 //     Duration q25 = getQuantilTick!0.25(ticks);
61 //     assert(q25 == dur!"seconds"(2), q25.toString());
62 
63 //     Duration q50 = getQuantilTick!0.50(ticks);
64 //     assert(q50 == dur!"seconds"(3), q25.toString());
65 
66 //     Duration q75 = getQuantilTick!0.75(ticks);
67 //     assert(q75 == dur!"seconds"(4), q25.toString());
68 
69 //     q25 = getQuantilTick!0.25(ticks[0 .. 4]);
70 //     assert(q25 == dur!"seconds"(1) + dur!"msecs"(500), q25.toString());
71 
72 //     q50 = getQuantilTick!0.50(ticks[0 .. 4]);
73 //     assert(q50 == dur!"seconds"(2) + dur!"msecs"(500), q25.toString());
74 
75 //     q75 = getQuantilTick!0.75(ticks[0 .. 4]);
76 //     assert(q75 == dur!"seconds"(3) + dur!"msecs"(500), q25.toString());
77 // }
78 
79 /** The options  controlling the behaviour of benchmark. */
80 struct BenchmarkOptions
81 {
82     string funcname; // the name of the function to benchmark
83     string filename; // the name of the file the results will be appended to
84     Duration duration = 1.seconds; // the time after which the function to
85                                    // benchmark is not executed anymore
86     size_t maxRounds = 10000; // the maximum number of times the function
87                               // to benchmark is called
88     int seed = 1337; // the seed to the random number generator
89 
90     this(string funcname)
91     {
92         this.funcname = funcname;
93     }
94 }
95 
96 /** This $(D struct) takes care of the time taking and outputting of the
97 statistics.
98 */
99 struct Benchmark
100 {
101     import std.array : Appender;
102 
103     string filename; // where to write the benchmark result to
104     string funcname; // the name of the benchmark
105     size_t rounds; // the number of times the functions is supposed to be
106     //executed
107     string timeScale; // the unit the benchmark is measuring in
108     real medianStopWatch; // the median time it takes to get the clocktime twice
109     bool dontWrite; // if set, no data is written to the the file name "filename"
110     // true if, RndValueGen opApply was interrupt unexpectitally
111     Appender!(Duration[]) ticks; // the stopped times, there will be rounds ticks
112     size_t ticksIndex = 0; // the index into ticks
113     size_t curRound = 0; // the number of rounds run
114     MonoTimeImpl!(ClockType.precise) startTime;
115     Duration timeSpend; // overall time spend running the benchmark function
116 
117     /** The constructor for the $(D Benchmark).
118     Params:
119         funcname = The name of the $(D benchmark) instance. The $(D funcname)
120             will be used to associate the results with the function
121         filename = The $(D filename) will be used as a filename to store the
122             results.
123     */
124     this(in string funcname, in size_t rounds, in string filename)
125     {
126         this.filename = filename;
127         this.funcname = funcname;
128         this.rounds = rounds;
129         this.timeScale = "hnsecs";
130         this.ticks = appender!(Duration[])();
131         this.medianStopWatch = medianStopWatchTime();
132     }
133 
134     /** A call to this method will start the time taking process */
135     void start()
136     {
137         this.startTime = MonoTimeImpl!(ClockType.precise).currTime;
138     }
139 
140     /** A call to this method will stop the time taking process, and
141     appends the execution time to the $(D ticks) member.
142     */
143     void stop()
144     {
145         auto end = MonoTimeImpl!(ClockType.precise).currTime;
146         Duration dur = end - this.startTime;
147         this.timeSpend += dur;
148         this.ticks.put(dur);
149         ++this.curRound;
150     }
151 
152     ~this()
153     {
154         import std.stdio : File;
155 
156         if (!this.dontWrite && this.ticks.data.length)
157         {
158             import std.algorithm : sort;
159 
160             auto sortedTicks = this.ticks.data;
161             sortedTicks.sort();
162 
163             auto f = File(filename ~ "_bechmark.csv", "a");
164             scope (exit)
165                 f.close();
166 
167             auto q0 = sortedTicks[0].total!("hnsecs")() /
168                 cast(double) this.rounds;
169             auto q25 = getQuantilTick!0.25(sortedTicks).total!("hnsecs")() /
170                 cast(double) this.rounds;
171             auto q50 = getQuantilTick!0.50(sortedTicks).total!("hnsecs")() /
172                    cast(double) this.rounds;
173             auto q75 = getQuantilTick!0.75(sortedTicks).total!("hnsecs")() /
174                 cast(double) this.rounds;
175             auto q100 = sortedTicks[$ - 1].total!("hnsecs")() /
176                 cast(double) this.rounds;
177 
178             // funcname, the data when the benchmark was created, unit of time,
179             // rounds, medianStopWatch, low, 0.25 quantil, median,
180             // 0.75 quantil, high
181             f.writefln(
182                 "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\""
183                 ~ ",\"%s\"",
184                 this.funcname, Clock.currTime.toISOExtString(),
185                 this.timeScale, this.curRound, this.medianStopWatch,
186                 q0 > this.medianStopWatch ? q0 - this.medianStopWatch : 0,
187                 q25 > this.medianStopWatch ? q25 - this.medianStopWatch : 0,
188                 q50 > this.medianStopWatch ? q50 - this.medianStopWatch : 0,
189                 q75 > this.medianStopWatch ? q75 - this.medianStopWatch : 0,
190                 q100 > this.medianStopWatch ? q100 - this.medianStopWatch : 0);
191         }
192     }
193 }
194 
195 void doNotOptimizeAway(T...)(ref T t)
196 {
197     foreach (ref it; t)
198     {
199         doNotOptimizeAwayImpl(&it);
200     }
201 }
202 
203 private void doNotOptimizeAwayImpl(void* p) {
204         import core.thread : getpid;
205         import std.stdio : writeln;
206         if(getpid() == 1) {
207                 writeln(*cast(char*)p);
208         }
209 }
210 
211 // unittest
212 // {
213 //     static void funToBenchmark(int a, float b, Gen!(int, -5, 5) c, string d,
214 //         GenASCIIString!(1, 10) e)
215 //     {
216 //         import core.thread;
217 
218 //         Thread.sleep(1.seconds / 100000);
219 //         doNotOptimizeAway(a, b, c, d, e);
220 //     }
221 
222 //     benchmark!funToBenchmark();
223 //     benchmark!funToBenchmark("Another Name");
224 //     benchmark!funToBenchmark("Another Name", 2.seconds);
225 //     benchmark!funToBenchmark(2.seconds);
226 // }
227 
228 /** This function runs the passed callable $(D T) for the duration of
229 $(D maxRuntime). It will count how often $(D T) is run in the duration and
230 how long each run took to complete.
231 
232 Unless compiled in release mode, statistics will be printed to $(D stderr).
233 If compiled in release mode the statistics are appended to a file called
234 $(D name).
235 
236 Params:
237     opts = A $(D BenchmarkOptions) instance that encompasses all possible
238         parameters of benchmark.
239     name = The name of the benchmark. The name is also used as filename to
240         save the benchmark results.
241     maxRuntime = The maximum time the benchmark is executed. The last run will
242         not be interrupted.
243     rndSeed = The seed to the random number generator used to populate the
244         parameter passed to the function to benchmark.
245     rounds = The maximum number of times the callable $(D T) is called.
246 */
247 void benchmark(alias T)(const ref BenchmarkOptions opts)
248 {
249         import std.random : Random;
250         import unit_threaded.randomized.random;
251 
252     auto bench = Benchmark(opts.funcname, opts.maxRounds, opts.filename);
253     auto rnd = Random(opts.seed);
254     enum string[] parameterNames = [ParameterIdentifierTuple!T];
255     auto valueGenerator = RndValueGen!(parameterNames, Parameters!T)(&rnd);
256 
257     while (bench.timeSpend <= opts.duration && bench.curRound < opts.maxRounds)
258     {
259         valueGenerator.genValues();
260 
261         bench.start();
262         try
263         {
264             T(valueGenerator.values);
265         }
266         catch (Throwable t)
267         {
268             import std.experimental.logger : logf;
269 
270             logf("unittest with name %s failed when parameter %s where passed",
271                 opts.funcname, valueGenerator);
272             break;
273         }
274         finally
275         {
276             bench.stop();
277             ++bench.curRound;
278         }
279     }
280 }
281 
282 /// Ditto
283 void benchmark(alias T)(string funcname = "", string filename = __FILE__)
284 {
285     import std.string : empty;
286 
287     auto opt = BenchmarkOptions(
288         funcname.empty ? fullyQualifiedName!T : funcname
289     );
290     opt.filename = filename;
291     benchmark!(T)(opt);
292 }
293 
294 /// Ditto
295 void benchmark(alias T)(Duration maxRuntime, string filename = __FILE__)
296 {
297     auto opt = BenchmarkOptions(fullyQualifiedName!T);
298     opt.filename = filename;
299     opt.duration = maxRuntime;
300     benchmark!(T)(opt);
301 }
302 
303 /// Ditto
304 /*void benchmark(alias T)(string name, string filename = __FILE__)
305 {
306     auto opt = BenchmarkOptions(name);
307     opt.filename = filename;
308     benchmark!(T)(opt);
309 }*/
310 
311 /// Ditto
312 void benchmark(alias T)(string name, Duration maxRuntime,
313     string filename = __FILE__)
314 {
315     auto opt = BenchmarkOptions(name);
316     opt.filename = filename;
317     opt.duration = maxRuntime;
318     benchmark!(T)(opt);
319 }