1 #ifndef LIBFASTER_DEFINITIONS_H 2 #define LIBFASTER_DEFINITIONS_H 8 #include <unordered_map> 11 #define UNUSED __attribute__((unused)) 21 #define LongInt 0x0004 25 #define POINTER 0x0100 28 #define LongIntP 0x0104 30 #define DoubleP 0x0110 35 #define LongIntV 0x1004 37 #define DoubleV 0x1010 38 #define GroupFDD 0x8000 43 #define OP_GENERICMAP 0x0100 45 #define OP_BulkMap 0x0102 46 #define OP_FlatMap 0x0104 47 #define OP_BulkFlatMap 0x0108 48 #define OP_MapByKey 0x0110 49 #define OP_FlatMapByKey 0x0120 50 #define OP_GENERICREDUCE 0x0200 51 #define OP_Reduce 0x0201 52 #define OP_BulkReduce 0x0202 53 #define OP_GENERICUPDATE 0x0400 54 #define OP_Update 0x0401 55 #define OP_UpdateByKey 0x0402 56 #define OP_BulkUpdate 0x0404 57 #define OP_GENERICMISC 0x0800 58 #define OP_CountByKey 0x0801 59 #define OP_GroupByKey 0x0802 60 #define OP_GroupByKeyH 0x0804 61 #define OP_CoGroup 0x0808 62 #define OP_Calibrate 0x0810 63 #define OP_OnlineRead 0x0811 64 #define OP_OnPartRead 0x0812 65 #define OP_OnFullPRead 0x0814 70 DiscardWorkerDL = 0x03,
78 GetLineSizesDL = 0x08,
99 ExchangeDataByKeyDL = 0x18,
100 GetKeyLocationDL= 0x19,
111 template <
typename T>
112 using onlineFullPartFuncP = int (*) (T & input);
114 template <
typename K,
typename T>
115 using IonlineFullPartFuncP = int (*) (K & key, T & input);
123 template <
typename T,
typename U>
124 using mapFunctionP = U (*) (T & input);
125 template <
typename T,
typename L,
typename U>
126 using ImapFunctionP = std::pair<L,U> (*) (T & input);
127 template <
typename T,
typename U>
128 using PmapFunctionP = std::pair<U,size_t> (*) (T & input);
129 template <
typename T,
typename L,
typename U>
130 using IPmapFunctionP = std::tuple<L,U,size_t> (*) (T & input);
133 template <
typename T,
typename U>
134 using bulkMapFunctionP = void (*) (U * output, T * input,
size_t size);
136 template <
typename T,
typename L,
typename U>
137 using IbulkMapFunctionP = void (*) (L * outKey, U * output, T * input,
size_t size);
139 template <
typename T,
typename U>
140 using PbulkMapFunctionP = void (*) (U * output,
size_t * outputDataSizes, T * input,
size_t size);
142 template <
typename T,
typename L,
typename U>
143 using IPbulkMapFunctionP = void (*) (L * outKey, U * output,
size_t * outputDataSizes, T * input,
size_t size);
149 template <
typename T,
typename U>
150 using flatMapFunctionP = std::deque<U> (*) (T & input);
151 template <
typename T,
typename L,
typename U>
152 using IflatMapFunctionP = std::deque<std::pair<L,U>> (*) (T & input);
153 template <
typename T,
typename U>
154 using PflatMapFunctionP = std::deque<std::pair<U,size_t>> (*) (T & input);
155 template <
typename T,
typename L,
typename U>
156 using IPflatMapFunctionP = std::deque<std::tuple<L, U,size_t>> (*) (T & input);
159 template <
typename T,
typename U>
160 using bulkFlatMapFunctionP = void (*) (U *& output,
size_t & outputSize, T * input,
size_t size);
162 template <
typename T,
typename L,
typename U>
163 using IbulkFlatMapFunctionP = void (*) (L *& outKey, U *& output,
size_t & outputSize, T * input,
size_t size);
165 template <
typename T,
typename U>
166 using PbulkFlatMapFunctionP = void (*) (U *& output,
size_t *& outputDataSizes,
size_t & outputSize, T * input,
size_t size);
168 template <
typename T,
typename L,
typename U>
169 using IPbulkFlatMapFunctionP = void (*) (L *& outKey, U *& output,
size_t *& outputDataSizes,
size_t & outputSize, T * input,
size_t size);
175 template <
typename T>
176 using reduceFunctionP = T (*) (T & a, T & b);
179 template <
typename T>
180 using bulkReduceFunctionP = T (*) (T * input,
size_t size);
188 template <
typename T,
typename U>
189 using mapPFunctionP = U (*) (T * input,
size_t size);
190 template <
typename T,
typename L,
typename U>
191 using ImapPFunctionP = std::pair<L,U> (*) (T * input,
size_t size);
192 template <
typename T,
typename U>
193 using PmapPFunctionP = std::pair<U,size_t> (*) (T * input,
size_t size);
194 template <
typename T,
typename L,
typename U>
195 using IPmapPFunctionP = std::tuple<L,U,size_t> (*) (T * input,
size_t size);
199 template <
typename T,
typename U>
200 using bulkMapPFunctionP = void (*) (U * output, T ** input,
size_t * inputDataSizes,
size_t size);
202 template <
typename T,
typename L,
typename U>
203 using IbulkMapPFunctionP = void (*) (L * outKey, U * output, T ** input,
size_t * inputDataSizes,
size_t size);
205 template <
typename T,
typename U>
206 using PbulkMapPFunctionP = void (*) (U * output,
size_t * outputDataSizes, T ** input,
size_t * inputDataSizes,
size_t size);
208 template <
typename T,
typename L,
typename U>
209 using IPbulkMapPFunctionP = void (*) (L * outKey, U * output,
size_t * outputDataSizes, T ** input,
size_t * inputDataSizes,
size_t size);
216 template <
typename T,
typename U>
217 using flatMapPFunctionP = std::deque<U> (*) (T *& input,
size_t size);
218 template <
typename T,
typename L,
typename U>
219 using IflatMapPFunctionP = std::deque<std::pair<L,U>> (*) (T *& input,
size_t size);
220 template <
typename T,
typename U>
221 using PflatMapPFunctionP = std::deque<std::pair<U, size_t>> (*) (T *& input,
size_t size);
222 template <
typename T,
typename L,
typename U>
223 using IPflatMapPFunctionP = std::deque<std::tuple<L, U, size_t>> (*) (T *& input,
size_t size);
227 template <
typename T,
typename U>
228 using bulkFlatMapPFunctionP = void (*) (U *& output,
size_t & outputSize, T ** input,
size_t * inputDataSizes,
size_t size) ;
230 template <
typename T,
typename L,
typename U>
231 using IbulkFlatMapPFunctionP = void (*) (L *& outKey, U *& output,
size_t & outputSize, T ** input,
size_t * inputDataSizes,
size_t size) ;
233 template <
typename T,
typename U>
234 using PbulkFlatMapPFunctionP = void (*) (U *& output,
size_t * outputDataSizes,
size_t & outputSize, T ** input,
size_t * inputDataSizes,
size_t size);
236 template <
typename T,
typename L,
typename U>
237 using IPbulkFlatMapPFunctionP = void (*) (L *& outKey, U *& output,
size_t * outputDataSizes,
size_t & outputSize, T ** input,
size_t * inputDataSizes,
size_t size);
245 template <
typename T>
246 using PreducePFunctionP = std::pair<T *, size_t> (*) (T * a,
size_t sizeA, T * b,
size_t sizeB);
249 template <
typename T>
250 using PbulkReducePFunctionP = std::pair<T *, size_t> (*) (T ** input,
size_t * inputDataSizes,
size_t size);
258 template <
typename K,
typename T>
259 using updateIFunctionP = void (*) (K & inKey, T & input);
261 template <
typename K,
typename T>
262 using updateByKeyIFunctionP = void (*) (K & inKey, std::vector<T *> & input);
268 template <
typename K,
typename T,
typename L,
typename U>
269 using ImapIFunctionP = std::pair<L,U> (*) (
const K & inKey, T & input);
270 template <
typename K,
typename T,
typename U>
271 using mapIFunctionP = U (*) (
const K & inKey, T & input);
272 template <
typename K,
typename T,
typename L,
typename U>
274 using IPmapIFunctionP = std::tuple<L,U,size_t> (*) (
const K & inKey, T & input);
275 template <
typename K,
typename T,
typename U>
276 using PmapIFunctionP = std::pair<U, size_t> (*) (
const K & inKey, T & input);
289 template <
typename K,
typename T,
typename L,
typename U>
290 using ImapByKeyIFunctionP = std::pair<L,U> (*) (
const K & inKey, std::vector<T *> & input);
292 template <
typename K,
typename T,
typename U>
293 using mapByKeyIFunctionP = U (*) (
const K & inKey, std::vector <T *> & input);
295 template <
typename K,
typename T,
typename L,
typename U>
296 using IPmapByKeyIFunctionP = std::tuple<L,U,size_t> (*) (
const K & inKey, std::vector <T *> & input);
298 template <
typename K,
typename T,
typename U>
299 using PmapByKeyIFunctionP = std::pair<U, size_t> (*) (
const K & inKey, std::vector <T *> & input);
302 template <
typename K,
typename T,
typename L,
typename U>
303 using IbulkMapIFunctionP = void (*) (L * outKey, U * output, K * inKey, T * input,
size_t size);
305 template <
typename K,
typename T,
typename U>
306 using bulkMapIFunctionP = void (*) (U * output, K * inKey, T * input,
size_t size);
308 template <
typename K,
typename T,
typename L,
typename U>
309 using IPbulkMapIFunctionP = void (*) (L * outKey, U * output,
size_t * outputDataSizes, K * inKey, T * input,
size_t size);
311 template <
typename K,
typename T,
typename U>
312 using PbulkMapIFunctionP = void (*) (U * output,
size_t * outputDataSizes, K * inKey, T * input,
size_t size);
318 template <
typename K,
typename T,
typename L,
typename U>
319 using IflatMapIFunctionP = std::deque<std::pair<L,U>> (*) (K inKey, T & input);
320 template <
typename K,
typename T,
typename U>
321 using flatMapIFunctionP = std::deque<U> (*) (K inKey, T & input);
322 template <
typename K,
typename T,
typename L,
typename U>
323 using IPflatMapIFunctionP = std::deque<std::tuple<L, U,size_t>> (*) (K inKey, T & input);
324 template <
typename K,
typename T,
typename U>
325 using PflatMapIFunctionP = std::deque<std::pair<U, size_t>> (*) (K inKey, T & input);
328 template <
typename K,
typename T,
typename L,
typename U>
329 using IbulkFlatMapIFunctionP = void (*) (L *& outKey, U *& output,
size_t & outputSize, K * inKey, T * input,
size_t size);
331 template <
typename K,
typename T,
typename U>
332 using bulkFlatMapIFunctionP = void (*) (U *& output,
size_t & outputSize, K * inKey, T * input,
size_t size);
334 template <
typename K,
typename T,
typename L,
typename U>
335 using IPbulkFlatMapIFunctionP = void (*) (L *& outKey, U *& output,
size_t *& outputDataSizes,
size_t & outputSize, K * inKey, T * input,
size_t size);
337 template <
typename K,
typename T,
typename U>
338 using PbulkFlatMapIFunctionP = void (*) (U *& output,
size_t *& outputDataSizes,
size_t & outputSize, K * inKey, T * input,
size_t size);
344 template <
typename K,
typename T>
345 using IreduceIFunctionP = std::pair<K,T> (*) (
const K & keyA, T & a,
const K & keyB, T & b);
348 template <
typename K,
typename T>
349 using IreduceByKeyIFunctionP = std::pair<K,T> (*) (
const K & keyA, T * a,
size_t sizeA,
const K & keyB, T * b,
size_t sizeB);
352 template <
typename K,
typename T>
353 using IbulkReduceIFunctionP = std::pair<K,T> (*) (K * key, T * input,
size_t size);
361 template <
typename K,
typename T,
typename L,
typename U>
362 using ImapIPFunctionP = std::pair<L,U> (*) (K inKey, T * input,
size_t size);
363 template <
typename K,
typename T,
typename U>
364 using mapIPFunctionP = U (*) (K inKey, T * input,
size_t size);
365 template <
typename K,
typename T,
typename L,
typename U>
366 using IPmapIPFunctionP = std::tuple<L,U,size_t> (*) (K inKey, T * input,
size_t size);
367 template <
typename K,
typename T,
typename U>
368 using PmapIPFunctionP = std::pair<U, size_t> (*) (K inKey, T * input,
size_t size);
371 template <
typename K,
typename T,
typename L,
typename U>
372 using ImapByKeyIPFunctionP = std::pair<L,U> (*) (
const K & inKey, std::vector<std::pair<T*,size_t>>);
374 template <
typename K,
typename T,
typename U>
375 using mapByKeyIPFunctionP = U (*) (
const K & inKey, std::vector<std::pair<T*,size_t>>);
377 template <
typename K,
typename T,
typename L,
typename U>
378 using IPmapByKeyIPFunctionP = std::tuple<L,U,size_t> (*) (
const K & inKey, std::vector<std::pair<T*,size_t>>);
380 template <
typename K,
typename T,
typename U>
381 using PmapByKeyIPFunctionP = std::pair<U, size_t> (*) (
const K & inKey, std::vector<std::pair<T*,size_t>>);
384 template <
typename K,
typename T,
typename L,
typename U>
385 using IbulkMapIPFunctionP = void (*) (L * outKey, U * output, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
387 template <
typename K,
typename T,
typename U>
388 using bulkMapIPFunctionP = void (*) (U * output, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
390 template <
typename K,
typename T,
typename L,
typename U>
391 using IPbulkMapIPFunctionP = void (*) (L * outKey, U * output,
size_t * outputDataSizes, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
393 template <
typename K,
typename T,
typename U>
394 using PbulkMapIPFunctionP = void (*) (U * output,
size_t * outputDataSizes, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
401 template <
typename K,
typename T,
typename L,
typename U>
402 using IflatMapIPFunctionP = std::deque<std::pair<L,U>> (*) (T *& input,
size_t size);
403 template <
typename K,
typename T,
typename U>
404 using flatMapIPFunctionP = std::deque<U> (*) (T *& input,
size_t size);
405 template <
typename K,
typename T,
typename L,
typename U>
406 using IPflatMapIPFunctionP = std::deque<std::tuple<L, U, size_t>> (*) (T *& input,
size_t size);
407 template <
typename K,
typename T,
typename U>
408 using PflatMapIPFunctionP = std::deque<std::pair<U, size_t>> (*) (T *& input,
size_t size);
412 template <
typename K,
typename T,
typename L,
typename U>
413 using IbulkFlatMapIPFunctionP = void (*) (L *& outKey, U *& output,
size_t & outputSize, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size) ;
415 template <
typename K,
typename T,
typename U>
416 using bulkFlatMapIPFunctionP = void (*) (U *& output,
size_t & outputSize, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size) ;
418 template <
typename K,
typename T,
typename L,
typename U>
419 using IPbulkFlatMapIPFunctionP = void (*) (L *& outKey, U *& output,
size_t * outputDataSizes,
size_t & outputSize, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
421 template <
typename K,
typename T,
typename U>
422 using PbulkFlatMapIPFunctionP = void (*) (U *& output,
size_t * outputDataSizes,
size_t & outputSize, K * inKey, T ** input,
size_t * inputDataSizes,
size_t size);
430 template <
typename K,
typename T>
431 using IPreduceIPFunctionP = std::tuple<K,T*,size_t> (*) (K keyA, T * a,
size_t sizeA, K keyB, T * b,
size_t sizeB);
434 template <
typename K,
typename T>
435 using IPreduceByKeyIPFunctionP = std::tuple<K,T*,size_t> (*) (K keyA, T ** a,
size_t * dataSizesA,
size_t sizeA, K keyB, T ** b,
size_t * dataSizesB,
size_t sizeB);
438 template <
typename K,
typename T>
439 using IPbulkReduceIPFunctionP = std::tuple<K,T*,size_t> (*) (K * key, T ** input,
size_t * inputDataSizes,
size_t size);
448 template <
typename K>
449 using updateByKeyG2FunctionP = void (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b);
452 template <
typename K>
453 using updateByKeyG3FunctionP = void (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b, std::vector<void*> & c);
457 template <
typename K>
458 using bulkUpdateG2FunctionP = void (*) (K * keyA,
void * a,
size_t na, K * keyB,
void * b,
size_t nb);
461 template <
typename K>
462 using bulkUpdateG3FunctionP = void (*) (K * keyA,
void * a,
size_t na, K * keyB,
void * b,
size_t nb, K * keyC,
void * c,
size_t nc);
469 template <
typename K,
typename To>
470 using mapByKeyG2FunctionP = To (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b);
472 template <
typename K,
typename To>
473 using mapByKeyG3FunctionP = To (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b, std::vector<void*> & c);
475 template <
typename K,
typename Ko,
typename To>
476 using ImapByKeyG2FunctionP = std::pair<Ko,To> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b);
478 template <
typename K,
typename Ko,
typename To>
479 using ImapByKeyG3FunctionP = std::pair<Ko,To> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b, std::vector<void*> & c);
487 template <
typename K,
typename To>
488 using flatMapByKeyG2FunctionP = std::deque<To> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b);
491 template <
typename K,
typename To>
492 using flatMapByKeyG3FunctionP = std::deque<To> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b, std::vector<void*> & c);
495 template <
typename K,
typename Ko,
typename To>
496 using IflatMapByKeyG2FunctionP = std::deque<std::pair<Ko,To>> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b);
499 template <
typename K,
typename Ko,
typename To>
500 using IflatMapByKeyG3FunctionP = std::deque<std::pair<Ko,To>> (*) (
const K & key, std::vector<void*> & a, std::vector<void*> & b, std::vector<void*> & c);
503 template <
typename K,
typename To>
504 using bulkFlatMapG2FunctionP = std::deque<To> (*) (K * keyA,
void* a,
size_t na, K * keyB,
void* b,
size_t nb);
507 template <
typename K,
typename To>
508 using bulkFlatMapG3FunctionP = std::deque<To> (*) (K * keyA,
void* a,
size_t na, K * keyB,
void* b,
size_t nb, K * keyC,
void* c,
size_t nc);
511 template <
typename K,
typename Ko,
typename To>
512 using IbulkFlatMapG2FunctionP = std::deque<std::pair<Ko,To>> (*) (K * keyA,
void* a,
size_t na, K * keyB,
void* b,
size_t nb);
515 template <
typename K,
typename Ko,
typename To>
516 using IbulkFlatMapG3FunctionP = std::deque<std::pair<Ko,To>> (*) (K * keyA,
void* a,
size_t na, K * keyB,
void* b,
size_t nb, K * keyC,
void* c,
size_t nc);
unsigned int fddOpType
Dataset operation type.
unsigned int fddType
Dataset type.