magpie.helper source code

1 module magpie.helper;
2 
3 import magpie.dataframe: DataFrame;
4 import magpie.index: Index;
5 import magpie.group: Group;
6 
7 import std.meta: AliasSeq, Repeat;
8 import std.range.primitives: ElementType;
9 import std.traits: isStaticArray, isIntegral, isFloatingPoint;
10 
11 /// Build DataFrame argument list
12 template getArgsList(args...)
13 {
14     static if(args.length)
15     {
16         alias arg = args[0];
17         import std.traits: isType;
18         static if(args.length == 1)
19         {
20             static if(isStaticArray!(arg))
21                 alias getArgsList = AliasSeq!(Repeat!(arg.length, ElementType!(arg)));
22             else
23                 alias getArgsList = AliasSeq!(arg);
24         }
25         else
26         {
27             static if(isType!(args[1]))
28             {
29                 static if(isStaticArray!(arg))
30                     alias getArgsList = AliasSeq!(Repeat!(arg.length, ElementType!(arg)), getArgsList!(args[1 .. $]));
31                 else
32                     alias getArgsList = AliasSeq!(arg, getArgsList!(args[1 .. $]));
33             }
34             else
35             {
36                 alias getArgsList = AliasSeq!(Repeat!(args[1],arg), getArgsList!(args[2 .. $]));
37             }
38         }
39     }
40     else
41         alias getArgsList = AliasSeq!();
42 }
43 
44 private template dropper_internal(int[] pos, int rem, Types...)
45 {
46     static if(pos.length == 0)
47     {
48         alias dropper_internal = Types;
49     }
50     else
51     {
52         alias dropper_internal = AliasSeq!(Types[0 .. pos[0] - rem],
53             dropper_internal!(pos[1 .. $], pos[0] + 1 ,Types[pos[0] - rem + 1 .. $]));
54     }
55 }
56 
57 /// Template to evaluate RowType - removes elements in Types according to the positions in pos. pos must be ascending
58 template dropper(int[] pos, Types...)
59 {
60     alias dropper = dropper_internal!(pos, 0, Types);
61 }
62 
63 /// drops values at a set of position from given array
64 T[] dropper(T)(int[] pos, T[] values)
65 {
66     T[] dropper_internal(T)(int[] pos, int rem, T[] values)
67     {
68         if(pos.length == 0)
69             return values;
70         else
71             return values[0 .. pos[0] - rem] ~ dropper_internal(pos[1 .. $], pos[0] + 1, values[pos[0] - rem + 1 .. $]);
72     }
73 
74     return dropper_internal(pos, 0, values);
75 }
76 
77 /// Function to sort indexes in ascending order and switch the code to keep the effective positions same
78 void sortIndex(string[] index, int[] codes)
79 {
80     foreach(i; 0 .. cast(uint)index.length)
81     {
82         uint pos = i;
83         foreach(j; i + 1 .. cast(uint)index.length)
84         {
85             if(index[pos] > index[j])
86             {
87                 pos = j;
88             }
89         }
90 
91         if(i == pos)
92             continue;
93 
94         string tmp = index[i];
95         index[i] = index[pos];
96         index[pos] = tmp;
97 
98         foreach(j; 0 .. codes.length)
99         {
100             if(codes[j] == pos)
101                 codes[j] = i;
102             else if(codes[j] == i)
103                 codes[j] = pos;
104         }
105     }
106 }
107 
108 /// Does exactly what generateCode does in Index but for any general ndarray
109 int[] vectorize(T)(T[] values)
110 {
111     int[T] elementPos;
112     int[] pos;
113     int totalUnique = 0;
114 
115     foreach(i; values)
116     {
117         ++pos.length;
118         pos[$ - 1] = elementPos.require(cast(immutable)i, { return totalUnique++; }());
119     }
120 
121     return totalUnique ~ pos;
122 }
123 
124 /// Transposes an array of integer
125 T[][] transposed(T)(T[][] data)
126 {
127     T[][] ret;
128     ret.length = data[0].length;
129     foreach(i, eleu; data)
130     {
131         foreach(j, elel; eleu)
132         {
133             ++ret[j].length;
134             ret[j][i] = elel;
135         }
136     }
137 
138     return ret;
139 }
140 
141 /// Template to get array from type
142 alias toArr(T) = T[];
143 
144 /// Get unique indexes for merge
145 Index ensureUnique(int axis)(Index i1, Index i2, string lsuffix = "x_", string rsuffix = "y_")
146 {
147     assert(i1.indexing[axis].codes.length == i2.indexing[axis].codes.length, "Index level mismatch for union");
148     
149     import std.algorithm: countUntil;
150     string[][] unique;
151 
152     unique.length = i1.indexing[axis].codes[0].length + i1.indexing[axis].codes[0].length;
153 
154     foreach(i; 0 .. i1.indexing[axis].codes[0].length)
155     {
156         string[] indx;
157         indx.length = i1.indexing[axis].codes.length;
158 
159         import std.range: lockstep;
160         foreach(j, a, b; lockstep(i1.indexing[axis].index, i1.indexing[axis].codes))
161         {
162             if(a.length == 0)
163             {
164                 import std.conv: to;
165                 indx[j] = to!string(b[i]);
166             }
167             else
168                 indx[j] = a[b[i]];
169         }
170 
171         // Making an asuumption that there is no conflicting indexes within the same Index struct
172         unique[i] = indx;
173     }
174 
175     foreach(i; 0 .. i2.indexing[axis].codes[0].length)
176     {
177         string[] indx;
178         indx.length = i2.indexing[axis].codes.length;
179 
180         import std.range: lockstep;
181         foreach(j, a, b; lockstep(i2.indexing[axis].index, i2.indexing[axis].codes))
182         {
183             if(a.length == 0)
184             {
185                 import std.conv: to;
186                 indx[j] = to!string(b[i]);
187             }
188             else
189                 indx[j] = a[b[i]];
190         }
191 
192         // Making an asuumption that there is no conflicting indexes within the same Index struct
193         const int p = cast(int)countUntil(unique, indx);
194         if(p < 0)
195             unique[i + i1.indexing[axis].codes[0].length] = indx;
196         else
197         {
198             foreach(j; 0 .. unique[p].length)
199                 unique[p][j] = lsuffix ~ unique[p][j];
200 
201             foreach(j; 0 .. indx.length)
202                 indx[j] = rsuffix ~ indx[j];
203 
204             unique[i + i1.indexing[axis].codes[0].length] = indx;
205         }
206     }
207 
208     Index ret;
209     ret.indexing[axis].index = transposed(unique);
210     ret.generateCodes();
211 
212     return ret;
213 }
214 
215 /// Returns the union of indexes in the order they appear
216 Index indexUnion(int axis)(Index i1, Index i2)
217 {
218     assert(i1.indexing[axis].codes.length == i2.indexing[axis].codes.length, "Index level mismatch for union");
219     
220     import std.algorithm: countUntil;
221     string[][] unique;
222 
223     foreach(i; 0 .. i1.indexing[axis].codes[0].length)
224     {
225         string[] indx;
226         indx.length = i1.indexing[axis].codes.length;
227 
228         import std.range: lockstep;
229         foreach(j, a, b; lockstep(i1.indexing[axis].index, i1.indexing[axis].codes))
230         {
231             if(a.length == 0)
232             {
233                 import std.conv: to;
234                 indx[j] = to!string(b[i]);
235             }
236             else
237                 indx[j] = a[b[i]];
238         }
239 
240         // Making an asuumption that there is no conflicting indexes within the same Index struct
241         unique ~= indx;
242     }
243 
244     foreach(i; 0 .. i2.indexing[axis].codes[0].length)
245     {
246         string[] indx;
247         indx.length = i2.indexing[axis].codes.length;
248 
249         import std.range: lockstep;
250         foreach(j, a, b; lockstep(i2.indexing[axis].index, i2.indexing[axis].codes))
251         {
252             if(a.length == 0)
253             {
254                 import std.conv: to;
255                 indx[j] = to!string(b[i]);
256             }
257             else
258                 indx[j] = a[b[i]];
259         }
260 
261         // Making an asuumption that there is no conflicting indexes within the same Index struct
262         const int p = cast(int)countUntil(unique, indx);
263         if(p < 0)
264             unique ~= indx;
265     }
266 
267     Index ret;
268     ret.indexing[axis].index = transposed(unique);
269     ret.generateCodes();
270 
271     return ret;
272 }
273 
274 /// find set of indexes that occur in both structures
275 Index indexIntersection(int axis)(Index i1, Index i2)
276 {
277     assert(i1.indexing[axis].codes.length == i2.indexing[axis].codes.length, "Index level mismatch for union");
278     
279     import std.algorithm: countUntil;
280     string[][] intersect;
281 
282     foreach(i; 0 .. i1.indexing[axis].codes[0].length)
283     {
284         string[] indx;
285         indx.length = i1.indexing[axis].codes.length;
286 
287         import std.range: lockstep;
288         foreach(j, a, b; lockstep(i1.indexing[axis].index, i1.indexing[axis].codes))
289         {
290             if(a.length == 0)
291             {
292                 import std.conv: to;
293                 indx[j] = to!string(b[i]);
294             }
295             else
296                 indx[j] = a[b[i]];
297         }
298 
299         // Making an asuumption that there is no conflicting indexes within the same Index struct
300         if(i2.getPosition!(axis)(indx) > -1)
301             intersect ~= indx;
302     }
303 
304     Index ret;
305     ret.indexing[axis].index = transposed(intersect);
306     ret.generateCodes();
307 
308     return ret;
309 }
310 
311 /// Template to check if the input is a DataFrame or not
312 template isDataFrame(T)
313 {
314     static if(is(T: DataFrame!Ty, Ty...))
315         enum bool isDataFrame = true;
316     else
317         enum bool isDataFrame = false;
318 }
319 
320 /// Template to check if the input is a Group or not
321 template isGroup(T)
322 {
323     static if(is(T: Group!Ty, Ty...))
324         enum bool isGroup = true;
325     else
326         enum bool isGroup = false;
327 }
328 
329 /// Check if given DataFrame is Homogeneous
330 template isHomogeneous(Args...)
331 {
332     static if(Args.length == 1)
333         enum bool isHomogeneous = true; 
334     else
335         enum bool isHomogeneous = is(Args[0] == Args[1]) && isHomogeneous!(Args[1 .. $]);
336 }
337 
338 /// Finding suitable type for aggregate
339 template suitableType(Types...)
340 {
341     import std.meta: anySatisfy;
342     import std.traits: isFloatingPoint, Largest;
343     static if(anySatisfy!(isFloatingPoint, Types))
344         alias suitableType = double;
345     else
346         alias suitableType = Largest!(Types);
347 }
348 
349 mixin template auxDispatch(alias F, bool isHomogeneousType, RowType...)
350 {
351     auto auxDispatch(size_t indx)
352     {
353         static if(isHomogeneousType)
354             return F(indx);
355         else
356             static foreach(i; 0 .. RowType.length)
357                 if(i == indx)
358                     return F!(i);
359 
360         assert(0);
361     }
362 }
363 
364 // Community suggested way ot intialize a DataFrame
365 unittest
366 {
367     assert(is(getArgsList!(int[2]) == AliasSeq!(int, int)));
368     assert(is(getArgsList!(int[2], double[3]) == AliasSeq!(int, int, double, double, double)));
369     assert(is(getArgsList!(int[1], double[2], int, 2) == AliasSeq!(int, double, double, int, int)));
370     assert(is(getArgsList!(int, int, double[2], int, 2) == AliasSeq!(int, int, double, double, int, int)));
371 }
372 
373 // Sorting indexes and codes by keeping their effet position same
374 unittest
375 {
376     string[] indx = ["b", "c", "a", "d"];
377     int[] codes = [0, 1, 2, 3];
378     sortIndex(indx, codes);
379     assert(indx == ["a", "b", "c", "d"]);
380     assert(codes == [1, 2, 0, 3]);
381 }
382 
383 // Testing dropper
384 unittest
385 {
386     assert(is(dropper!([1, 4], int, long, int, long, byte, float, bool) == AliasSeq!(int, int, long, float, bool)));
387     assert(is(dropper!([0, 4], int, long, int, long, double) == AliasSeq!(long, int, long)));
388     assert(is(dropper!([1, 3, 5], int, long, int, long, double, float, bool) == AliasSeq!(int, int, double, bool)));
389     assert(is(dropper!([0, 2, 3, 5], int, uint, byte, ubyte, long, ulong, bool) == AliasSeq!(uint, long, bool)));
390 }
391 
392 // Testing dropper for arrays
393 unittest
394 {
395     assert(dropper([1, 4], [1, 2, 3, 4, 5, 6]) == [1, 3, 4, 6]);
396     assert(dropper([0, 5], [1, 2, 3, 4, 5, 6]) == [2, 3, 4, 5]);
397     assert(dropper([0, 3, 5], [1, 2, 3, 4, 5, 6]) == [2, 3, 5]);
398 }
399 
400 // Vectorize test
401 unittest
402 {
403     double[] arr = [1.2, 2.7, 1.2, 5.6, 1.2, 5.6];
404     auto varr = vectorize(arr);
405     assert(varr.length == arr.length + 1);
406     assert(varr[0] == 3);
407     assert(varr[1 .. $] == [0, 1, 0, 2, 0, 2]);
408 
409     string[] arrs = ["Hello", "Hi", "Hello", "Hi"];
410     auto varrs = vectorize(arrs);
411     assert(varrs.length == arrs.length + 1);
412     assert(varrs[0] == 2);
413     assert(varrs[1 .. $] == [0, 1, 0, 1]);
414 
415     int[][] arr2d = [[1, 2], [2, 4], [1, 2], [3, 6]];
416     auto varr2d = vectorize(arr2d);
417     assert(varr2d.length == arr2d.length + 1);
418     assert(varr2d[0] == 3);
419     assert(varr2d[1 .. $] == [0, 1, 0, 2]);
420 }
421 
422 // Transposed test
423 unittest
424 {
425     int[][] a = [[1, 2, 3], [4, 5, 6]];
426     assert(transposed(a) == [[1, 4], [2, 5], [3, 6]]);
427 
428     int[][] b = [[1, 2, 3], [4, 5, 6], [7, 8, 9]];
429     assert(transposed(b) == [[1, 4, 7], [2, 5, 8], [3, 6, 9]]);
430 }
431 
432 
433 // Ensuring unque indexes on merge
434 unittest
435 {
436     Index i1;
437     Index i2;
438 
439     i1.setIndex([["Hello", "Hi"], ["Hi", "Hello"]], ["1", "@"]);
440     i2.setIndex([["Hey", "Yo"], ["Yo", "Hey"]], ["!", "2"]);
441 
442     auto unique = ensureUnique!0(i1, i2);
443     assert(unique.row.index == [["Hello", "Hey", "Hi", "Yo"], ["Hello", "Hey", "Hi", "Yo"]]);
444     assert(unique.row.codes == [[0, 2, 1, 3], [2, 0, 3, 1]]);
445     
446     i2.setIndex([["Hey", "Hello"], ["Hi", "Hi"]], ["!", "@"]);
447     unique = ensureUnique!0(i1, i2);
448     assert(unique.row.index == [["Hey", "Hi", "x_Hello", "y_Hello"], ["Hello", "Hi", "x_Hi", "y_Hi"]]);
449     assert(unique.row.codes == [[2, 1, 0, 3], [2, 0, 1, 3]]);
450 }
451 
452 // Index union
453 unittest
454 {
455     Index i1;
456     Index i2;
457 
458     i1.setIndex([["Hello", "Hi"], ["Hi", "Hello"]], ["1", "@"]);
459     i2.setIndex([["Hi", "Hello"], ["Yo", "Hi"]], ["!", "2"]);
460 
461     Index indxunion = indexUnion!0(i1, i2);
462     assert(indxunion.row.index == [["Hello", "Hi"], ["Hello", "Hi", "Yo"]]);
463     assert(indxunion.row.codes == [[0, 1, 1], [1, 0, 2]]);
464 }
465 
466 // Index Intersection
467 unittest
468 {
469     Index i1;
470     Index i2;
471 
472     i1.setIndex([["Hello", "Hi"], ["Hi", "Hello"]], ["1", "@"]);
473     i2.setIndex([["Hi", "Hello"], ["Yo", "Hi"]], ["!", "2"]);
474 
475     auto indxintersect =  indexIntersection!0(i1, i2);
476     assert(indxintersect.row.index == [["Hello"], ["Hi"]]);
477     assert(indxintersect.row.codes == [[0], [0]]);
478 }
479 
480 // Index Intersection
481 unittest
482 {
483     Index i1;
484     Index i2;
485 
486     i1.setIndex([["Hello", "Hi", "Hey"], ["Hi", "Hello", "Hey"], ["Hey", "Hi", "Hello"]], ["1", "@"]);
487     i2.setIndex([["Hi", "Hello"], ["Hello", "Hi"], ["Hi", "Hey"]], ["!", "2"]);
488 
489     auto indxintersect =  indexIntersection!0(i1, i2);
490     assert(indxintersect.row.index == [["Hello", "Hi"], ["Hello", "Hi"], ["Hey", "Hi"]]);
491     assert(indxintersect.row.codes == [[0, 1], [1, 0], [0, 1]]);
492 }
493 
494 // isDataFrame
495 unittest
496 {
497     static assert(isDataFrame!(DataFrame!(int, 2)) == true);
498     static assert(isDataFrame!(bool) == false);
499 }
500 
501 unittest
502 {
503     static assert(isHomogeneous!(int, int, int) == true);
504     static assert(isHomogeneous!(int, int, double) == false);
505 }