Data_transfer_lib/Butterfly.h at master · zhang-cheng09/Data_transfer_lib · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

/***************************************************************************
 * This is a source file of the Adaptive Data Transfer Library version 1.0
 * This file was initially finished by Cheng Zhang
 * If you have any problem,
 * please contact Cheng Zhang via zhang-cheng09@mails.tsinghua.edu.cn
 **************************************************************************/

#ifndef BUTTERFLY_H
#define BUTTERFLY_H
#include "mpi.h"
#include <vector>
#include <iostream>
using namespace std;

#define SEND 0
#define RECV 1
#define SENDRECV 2

class Butterfly
{
private:
    double best_timer;
    MPI_Comm intra_comm;
    bool profiling;
    bool * stage_mask;
    int current_stage;

    MPI_Comm global_comm;
    MPI_Comm local_comm;
    int * send_cells;
    int * recv_cells;
    int remote_model_size;
    int action;

    int local_model_size;
    int * recv_model_global_rank;
    int * send_model_global_rank;
    int global_rank;
    int local_rank;
    bool is_send;
    bool is_recv;

    int send_model_size;
    int recv_model_size;
    int send_node_index;
    int recv_node_index;
    vector<int> * send_node_per_proc_local_rank;
    vector<int> * recv_node_per_proc_local_rank;
    int * total_node_root_proc_global_rank;
    int send_node_num;
    int recv_node_num;
    int total_node_num;

    int * total_send_cells;
    int * total_recv_cells;

    bool send_node_master;
    bool recv_node_master;

    int fields_per_cell;
    int pre_fields_per_cell;

    int max_cells_cnts;

    int intra_node_recv_cells_cnts;
    int butterfly_node_num;
    int butterfly_grp_node_num;
    int * butterfly_node_global_rank;
    int send_node_map_to_butterfly_node;
    int send_node_send_cells_cnts;
    vector<int> * butterfly_node_map_to_send_node;
    vector<int> * butterfly_node_recv_cells_cnts;
    bool butterfly_node_master;
    int butterfly_node_index;
    int * send_node_cells_cnts;

    vector<int> * butterfly_node_map_to_recv_node;
    vector<int> * recv_node_map_to_butterfly_node;

    char * butterfly_data_buf[3];
    int butterfly_stage_num;
    int butterfly_grp_index;
    int butterfly_grp_node_index;
    int * recv_proc_node_index;
    vector<int> * butterfly_node_decompsitions;
    vector<int> * butterfly_node_decompsitions_cells;
    int * butterfly_pair_node_list;

    int butterfly_grp_num;
    char * butterfly_gather_buf;

    int * intra_proc_displs_cells;

    FILE * file;
    int * last_p2p_send_model_proc_lists;
    int last_p2p_stage_num;
    vector<int> last_p2p_send_to_remote_procs;
    vector<int> last_p2p_recv_from_remote_procs;
    int * last_p2p_send_cells;
    int * last_p2p_send_displs;
    int * last_p2p_recv_cells;
    char * last_p2p_recv_buf;
    char * last_p2p_send_buf;
    int last_p2p_recv_buf_cells;
    vector<int> last_p2p_send_mask;
    vector<int> last_p2p_send_mask_cells;

    int butterfly_p2p_stage_num;
    int p2p_depth;
    int * butterfly_p2p_remote_procs;
    int * butterfly_p2p_send_cells;
    int * butterfly_p2p_recv_cells;
    int * butterfly_p2p_send_displs;
    int * butterfly_p2p_recv_displs;
    vector<int> * butterfly_p2p_send_mask;
    vector<int> * butterfly_p2p_send_mask_cells;

    int * p2p_num_per_butterfly_stage;
    int * p2p_depth_per_butterfly_stage;

    int cur_buf_index;
    int local_buf_index;
    int remote_buf_index;

    int send_node_map_to_butterfly_node_index;
    int * first_p2p_sendcells;
    int * first_p2p_recvcells;
    int * first_p2p_senddispls;
    int * first_p2p_recvdispls;
    int first_p2p_send_proc_num;
    int first_p2p_recv_proc_num;
    int * first_p2p_send_mask;
    int * first_p2p_send_mask_cells;
    int * first_p2p_send_proc_index;
    int * first_p2p_recv_proc_index;

    void compute_butterfly_recv_rank(int *, int, int);
    void intra_node_gather_init(MPI_Comm, MPI_Comm, int *, int *, int *, int, int);
    void process_mapping_from_send_to_butterfly_init();
    void process_mapping_from_butterfly_to_recv_init();
    void decrease_sort(int *, int *, int);
    void butterfly_init();
    void last_p2p_init();
    void first_p2p_init();
    void wtime(double *);
    void butterfly_p2p_init(int * = NULL);

public:
    void butterfly_p2p_execute();
    void first_p2p_execute(char *, char *);
    void last_p2p_execute(char *);
    void execute(char *, char *, int);
    void reset_p2p_stage_num(int *);
    int * auto_set_p2p_stage_num(char *, char *, int);
    Butterfly(MPI_Comm, MPI_Comm, int *, int *, int *, int, int);
    ~Butterfly();
};
#endif