magpie.operation source code

1 module magpie.operation;
2 
3 /*
4  * This file will contain work on operations of aggregate and join
5  * aggregate:   Operation to work on a row/ column of DataFrame to give results like mean/median/mode/elc
6  * merge:       It joins two DataFrame in a way similar to that in Pandas or any RDBMS software
7  */
8 
9 import magpie.dataframe: DataFrame;
10 import magpie.group: Group;
11 import magpie.helper: isDataFrame, isGroup;
12 
13 /// Enums for joins
14 enum JoinTypes
15 {
16     left = 0,
17     right = 1,
18     outer = 2,
19     inner = 3,
20 }
21 
22 alias Left = JoinTypes.left;
23 alias Right = JoinTypes.right;
24 alias Inner = JoinTypes.inner;
25 alias Outer = JoinTypes.outer;
26 
27 /++
28 Description: Merges two DataFrames based on indexes
29 @params: type - Type of join: Inner, Outer, Left, Right
30 @params: df1 - The left DataFrame
31 @params: df2 - The right DataFrame
32 @params: lsuffix - Suffix to add in case there is an index collision between left and right DataFrame
33 @parmas: rsuffix - Suffix to add in case there is an index collision between left and right DataFrame
34 +/
35 auto merge(JoinTypes type = Inner, T, U)(T df1, U df2, string lsuffix = "x_", string rsuffix = "y_")
36     if(isDataFrame!T && isDataFrame!U)
37 {
38 
39     assert(df1.indx.row.codes.length == df2.indx.row.codes.length,
40         "Index level mismatch. Cannot merge dataframes with different level of indexing");
41 
42     if(df1.indx.column.codes.length > df2.indx.column.codes.length)
43     {
44         int[] codes;
45         codes.length = df2.indx.column.codes[0].length;
46 
47         foreach(i; 0 .. df1.indx.column.codes.length - df2.indx.column.codes.length)
48         {
49             df2.indx.column.index ~= [[""]];
50             df2.indx.column.codes ~= [codes];
51         }
52     }
53     else if(df1.indx.column.codes.length < df2.indx.column.codes.length)
54     {
55         int[] codes;
56         codes.length = df1.indx.column.codes[0].length;
57         
58         foreach(i; 0 .. df2.indx.column.codes.length - df1.indx.column.codes.length)
59         {
60             df1.indx.column.index ~= [[""]];
61             df1.indx.column.codes ~= [codes];
62         }
63     }
64 
65     DataFrame!(true, df1.RowType, df2.RowType) combinator;
66 
67     import magpie.helper: ensureUnique, transposed;
68     combinator.indx.column = ensureUnique!1(df1.indx, df2.indx, lsuffix, rsuffix).column;
69 
70     string[] unpack(size_t i)
71     {
72         string[] indx;
73         indx.length = combinator.indx.row.codes.length;
74 
75         import std.range: lockstep;
76         foreach(j, a, b; lockstep(combinator.indx.row.index, combinator.indx.row.codes))
77         {
78             if(a.length == 0)
79             {
80                 import std.conv: to;
81                 indx[j] = to!string(b[i]);
82             }
83             else
84             {
85                 indx[j] = a[b[i]];
86             }
87         }
88 
89         return indx;
90     }
91 
92     static if(type == JoinTypes.left)
93     {
94         combinator.indx.row = df1.indx.row;
95         static foreach(i; 0 .. df1.RowType.length)
96             combinator.data[i] = df1.data[i];
97         static foreach(i; 0 .. df2.RowType.length)
98             combinator.data[df1.RowType.length + i].length = df1.rows;
99 
100         foreach(i; 0 .. combinator.indx.row.codes[0].length)
101         {
102             string[] indx = unpack(i);
103             ptrdiff_t pos = df2.indx.getPosition!0(indx);
104 
105             if(pos > -1)
106             {
107                 static foreach(j; 0 .. df2.RowType.length)
108                 {
109                     combinator.data[df1.RowType.length + j][i] = df2.data[j][pos];
110                 }
111             }
112         }
113 
114         combinator.rows = df1.rows;
115         return combinator;
116     }
117     else static if(type == JoinTypes.right)
118     {
119         combinator.indx.row = df2.indx.row;
120         static foreach(i; 0 .. df2.RowType.length)
121             combinator.data[df1.RowType.length + i] = df2.data[i];
122         static foreach(i; 0 .. df1.RowType.length)
123             combinator.data[i].length = df2.rows;
124 
125         foreach(i; 0 .. combinator.indx.row.codes[0].length)
126         {
127             string[] indx = unpack(i);
128             ptrdiff_t pos = df1.indx.getPosition!0(indx);
129 
130             if(pos > -1)
131             {
132                 static foreach(j; 0 .. df1.RowType.length)
133                     combinator.data[j][i] = df1.data[j][pos];
134             }
135         }
136 
137         combinator.rows = df2.rows;
138         return combinator;
139     }
140     else static if(type == JoinTypes.outer)
141     {
142         import magpie.helper: indexUnion;
143         combinator.indx.row = indexUnion!0(df1.indx, df2.indx).row;
144         combinator.indx.row.titles = df1.indx.row.titles;
145 
146         combinator.rows = combinator.indx.row.codes[0].length;
147 
148         static foreach(i; 0 .. combinator.RowType.length)
149             combinator.data[i].length = combinator.rows;
150 
151         foreach(i; 0 .. combinator.rows)
152         {
153             string[] indx = unpack(i);
154 
155             ptrdiff_t pos = df1.indx.getPosition!0(indx);
156             if(pos > -1)
157             {
158                 static foreach(j; 0 .. df1.RowType.length)
159                     combinator.data[j][i] = df1.data[j][pos];
160             }
161 
162             pos = df2.indx.getPosition!0(indx);
163             if(pos > -1)
164             {
165                 static foreach(j; 0 .. df2.RowType.length)
166                     combinator.data[df1.RowType.length + j][i] = df2.data[j][pos];
167             }
168         }
169 
170         return combinator;
171     }
172     else static if(type == JoinTypes.inner)
173     {
174         import magpie.helper: indexIntersection;
175         combinator.indx.row = indexIntersection!0(df1.indx, df2.indx).row;
176         combinator.indx.row.titles = df1.indx.row.titles;
177 
178         combinator.rows = combinator.indx.row.codes[0].length;
179 
180         static foreach(i; 0 .. combinator.RowType.length)
181             combinator.data[i].length = combinator.rows;
182 
183         foreach(i; 0 .. combinator.rows)
184         {
185             string[] indx = unpack(i);
186 
187             ptrdiff_t pos = df1.indx.getPosition!0(indx);
188             if(pos > -1)
189             {
190                 static foreach(j; 0 .. df1.RowType.length)
191                     combinator.data[j][i] = df1.data[j][pos];
192             }
193 
194             pos = df2.indx.getPosition!0(indx);
195             if(pos > -1)
196             {
197                 static foreach(j; 0 .. df2.RowType.length)
198                     combinator.data[df1.RowType.length + j][i] = df2.data[j][pos];
199             }
200         }
201 
202         return combinator;
203     }
204     else static assert(0, "Invalid join type. Available join type: left, right, outer, inner(default)");
205 }
206 
207 // Left Join
208 unittest
209 {
210     import magpie.index: Index;
211 
212     DataFrame!(int, 2) df1;
213     DataFrame!(double, 2) df2;
214 
215     Index i1, i2;
216     i1.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"]]);
217     i2.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
218 
219     df1.setFrameIndex(i1);
220     df1.assign!1(0, [1, 2]);
221     df2.setFrameIndex(i2);
222     df2.assign!1(0, [1.0, 2.0]);
223 
224     assert(merge!(JoinTypes.right)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
225         ~ "I1                Hey    Hey\n"
226         ~ "Hello  1      0   1      nan\n"
227         ~ "Hi     2      0   2      nan\n"
228     );
229 }
230 
231 // Left Join
232 unittest
233 {
234     import magpie.index: Index;
235 
236     DataFrame!(int, 2) df1;
237     DataFrame!(double, 2) df2;
238 
239     Index i1, i2;
240     i1.setIndex([["Hello", "Hey"]], ["I1"], [["Hello", "Hi"]]);
241     i2.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
242 
243     df1.setFrameIndex(i1);
244     df1.assign!1(0, [1, 2]);
245     df2.setFrameIndex(i2);
246     df2.assign!1(0, [1.0, 2.0]);
247 
248     assert(merge!(JoinTypes.left)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
249         ~ "I1                Hey    Hey\n"
250         ~ "Hello  1      0   1      nan\n"
251         ~ "Hey    2      0   nan    nan\n"
252     );
253 }
254 
255 // Right Join
256 unittest
257 {
258     import magpie.index: Index;
259 
260     DataFrame!(int, 2) df1;
261     DataFrame!(double, 2) df2;
262 
263     Index i1, i2;
264     i1.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"]]);
265     i2.setIndex([["Hello", "Hey"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
266 
267     df1.setFrameIndex(i1);
268     df1.assign!1(0, [1, 2]);
269     df2.setFrameIndex(i2);
270     df2.assign!1(0, [1.0, 2.0]);
271 
272     assert(merge!(JoinTypes.right)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
273         ~ "I1                Hey    Hey\n"
274         ~ "Hello  1      0   1      nan\n"
275         ~ "Hey    0      0   2      nan\n"
276     );
277 }
278 
279 // Outer join
280 unittest
281 {
282     import magpie.index: Index;
283 
284     DataFrame!(int, 2) df1;
285     DataFrame!(double, 2) df2;
286 
287     Index i1, i2;
288     i1.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"]]);
289     i2.setIndex([["Hello", "Hey"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
290 
291     df1.setFrameIndex(i1);
292     df1.assign!1(0, [1, 2]);
293     df2.setFrameIndex(i2);
294     df2.assign!1(0, [1.0, 2.0]);
295 
296     assert(merge!(JoinTypes.outer)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
297         ~ "I1                Hey    Hey\n"
298         ~ "Hello  1      0   1      nan\n"
299         ~ "Hi     2      0   nan    nan\n"
300         ~ "Hey    0      0   2      nan\n"
301     );
302 }
303 
304 // Outer join
305 unittest
306 {
307     import magpie.index: Index;
308 
309     DataFrame!(int, 2) df1;
310     DataFrame!(double, 2) df2;
311 
312     Index i1, i2;
313     i1.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"]]);
314     i2.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
315 
316     df1.setFrameIndex(i1);
317     df1.assign!1(0, [1, 2]);
318     df2.setFrameIndex(i2);
319     df2.assign!1(0, [1.0, 2.0]);
320 
321     assert(merge!(JoinTypes.outer)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
322         ~ "I1                Hey    Hey\n"
323         ~ "Hello  1      0   1      nan\n"
324         ~ "Hi     2      0   2      nan\n"
325     );
326 }
327 
328 // Inner Join
329 unittest
330 {
331     import magpie.index: Index;
332 
333     DataFrame!(int, 2) df1;
334     DataFrame!(double, 2) df2;
335 
336     Index i1, i2;
337     i1.setIndex([["Hello", "Hi"]], ["I1"], [["Hello", "Hi"]]);
338     i2.setIndex([["Hello", "Hey"]], ["I1"], [["Hello", "Hi"], ["Hey", "Hey"]]);
339 
340     df1.setFrameIndex(i1);
341     df1.assign!1(0, [1, 2]);
342     df2.setFrameIndex(i2);
343     df2.assign!1(0, [1.0, 2.0]);
344 
345     assert(merge!(JoinTypes.inner)(df1, df2).display(true, 200) == "       Hello  Hi  Hello  Hi \n"
346         ~ "I1                Hey    Hey\n"
347         ~ "Hello  1      0   1      nan\n"
348     );
349 }