|
| 1 | +package destination |
| 2 | + |
| 3 | +import ( |
| 4 | + "bytes" |
| 5 | + "context" |
| 6 | + "encoding/json" |
| 7 | + "fmt" |
| 8 | + "io" |
| 9 | + |
| 10 | + "github.com/apache/arrow/go/v13/arrow" |
| 11 | + "github.com/apache/arrow/go/v13/arrow/ipc" |
| 12 | + pb "github.com/cloudquery/plugin-pb-go/pb/destination/v1" |
| 13 | + "github.com/cloudquery/plugin-pb-go/specs" |
| 14 | + "github.com/cloudquery/plugin-sdk/v3/plugins/destination" |
| 15 | + "github.com/cloudquery/plugin-sdk/v3/schema" |
| 16 | + "github.com/rs/zerolog" |
| 17 | + "golang.org/x/sync/errgroup" |
| 18 | + "google.golang.org/grpc/codes" |
| 19 | + "google.golang.org/grpc/status" |
| 20 | +) |
| 21 | + |
| 22 | +type Server struct { |
| 23 | + pb.UnimplementedDestinationServer |
| 24 | + Plugin *destination.Plugin |
| 25 | + Logger zerolog.Logger |
| 26 | + spec specs.Destination |
| 27 | +} |
| 28 | + |
| 29 | +func (s *Server) Configure(ctx context.Context, req *pb.Configure_Request) (*pb.Configure_Response, error) { |
| 30 | + var spec specs.Destination |
| 31 | + if err := json.Unmarshal(req.Config, &spec); err != nil { |
| 32 | + return nil, status.Errorf(codes.InvalidArgument, "failed to unmarshal spec: %v", err) |
| 33 | + } |
| 34 | + s.spec = spec |
| 35 | + return &pb.Configure_Response{}, s.Plugin.Init(ctx, s.Logger, spec) |
| 36 | +} |
| 37 | + |
| 38 | +func (s *Server) GetName(context.Context, *pb.GetName_Request) (*pb.GetName_Response, error) { |
| 39 | + return &pb.GetName_Response{ |
| 40 | + Name: s.Plugin.Name(), |
| 41 | + }, nil |
| 42 | +} |
| 43 | + |
| 44 | +func (s *Server) GetVersion(context.Context, *pb.GetVersion_Request) (*pb.GetVersion_Response, error) { |
| 45 | + return &pb.GetVersion_Response{ |
| 46 | + Version: s.Plugin.Version(), |
| 47 | + }, nil |
| 48 | +} |
| 49 | + |
| 50 | +func (s *Server) Migrate(ctx context.Context, req *pb.Migrate_Request) (*pb.Migrate_Response, error) { |
| 51 | + schemas, err := schema.NewSchemasFromBytes(req.Tables) |
| 52 | + if err != nil { |
| 53 | + return nil, status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) |
| 54 | + } |
| 55 | + tables, err := schema.NewTablesFromArrowSchemas(schemas) |
| 56 | + if err != nil { |
| 57 | + return nil, status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) |
| 58 | + } |
| 59 | + s.setPKsForTables(tables) |
| 60 | + |
| 61 | + return &pb.Migrate_Response{}, s.Plugin.Migrate(ctx, tables) |
| 62 | +} |
| 63 | + |
| 64 | +// Note the order of operations in this method is important! |
| 65 | +// Trying to insert into the `resources` channel before starting the reader goroutine will cause a deadlock. |
| 66 | +func (s *Server) Write(msg pb.Destination_WriteServer) error { |
| 67 | + resources := make(chan arrow.Record) |
| 68 | + |
| 69 | + r, err := msg.Recv() |
| 70 | + if err != nil { |
| 71 | + if err == io.EOF { |
| 72 | + return msg.SendAndClose(&pb.Write_Response{}) |
| 73 | + } |
| 74 | + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) |
| 75 | + } |
| 76 | + |
| 77 | + schemas, err := schema.NewSchemasFromBytes(r.Tables) |
| 78 | + if err != nil { |
| 79 | + return status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) |
| 80 | + } |
| 81 | + tables, err := schema.NewTablesFromArrowSchemas(schemas) |
| 82 | + if err != nil { |
| 83 | + return status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) |
| 84 | + } |
| 85 | + var sourceSpec specs.Source |
| 86 | + if r.SourceSpec == nil { |
| 87 | + // this is for backward compatibility |
| 88 | + sourceSpec = specs.Source{ |
| 89 | + Name: r.Source, |
| 90 | + } |
| 91 | + } else { |
| 92 | + if err := json.Unmarshal(r.SourceSpec, &sourceSpec); err != nil { |
| 93 | + return status.Errorf(codes.InvalidArgument, "failed to unmarshal source spec: %v", err) |
| 94 | + } |
| 95 | + } |
| 96 | + syncTime := r.Timestamp.AsTime() |
| 97 | + s.setPKsForTables(tables) |
| 98 | + eg, ctx := errgroup.WithContext(msg.Context()) |
| 99 | + eg.Go(func() error { |
| 100 | + return s.Plugin.Write(ctx, sourceSpec, tables, syncTime, resources) |
| 101 | + }) |
| 102 | + |
| 103 | + for { |
| 104 | + r, err := msg.Recv() |
| 105 | + if err == io.EOF { |
| 106 | + close(resources) |
| 107 | + if err := eg.Wait(); err != nil { |
| 108 | + return status.Errorf(codes.Internal, "write failed: %v", err) |
| 109 | + } |
| 110 | + return msg.SendAndClose(&pb.Write_Response{}) |
| 111 | + } |
| 112 | + if err != nil { |
| 113 | + close(resources) |
| 114 | + if wgErr := eg.Wait(); wgErr != nil { |
| 115 | + return status.Errorf(codes.Internal, "failed to receive msg: %v and write failed: %v", err, wgErr) |
| 116 | + } |
| 117 | + return status.Errorf(codes.Internal, "failed to receive msg: %v", err) |
| 118 | + } |
| 119 | + rdr, err := ipc.NewReader(bytes.NewReader(r.Resource)) |
| 120 | + if err != nil { |
| 121 | + close(resources) |
| 122 | + if wgErr := eg.Wait(); wgErr != nil { |
| 123 | + return status.Errorf(codes.InvalidArgument, "failed to create reader: %v and write failed: %v", err, wgErr) |
| 124 | + } |
| 125 | + return status.Errorf(codes.InvalidArgument, "failed to create reader: %v", err) |
| 126 | + } |
| 127 | + for rdr.Next() { |
| 128 | + rec := rdr.Record() |
| 129 | + rec.Retain() |
| 130 | + select { |
| 131 | + case resources <- rec: |
| 132 | + case <-ctx.Done(): |
| 133 | + close(resources) |
| 134 | + if err := eg.Wait(); err != nil { |
| 135 | + return status.Errorf(codes.Internal, "Context done: %v and failed to wait for plugin: %v", ctx.Err(), err) |
| 136 | + } |
| 137 | + return status.Errorf(codes.Internal, "Context done: %v", ctx.Err()) |
| 138 | + } |
| 139 | + } |
| 140 | + if err := rdr.Err(); err != nil { |
| 141 | + return status.Errorf(codes.InvalidArgument, "failed to read resource: %v", err) |
| 142 | + } |
| 143 | + } |
| 144 | +} |
| 145 | + |
| 146 | +func setCQIDAsPrimaryKeysForTables(tables schema.Tables) { |
| 147 | + for _, table := range tables { |
| 148 | + for i, col := range table.Columns { |
| 149 | + table.Columns[i].PrimaryKey = col.Name == schema.CqIDColumn.Name |
| 150 | + } |
| 151 | + setCQIDAsPrimaryKeysForTables(table.Relations) |
| 152 | + } |
| 153 | +} |
| 154 | + |
| 155 | +func (s *Server) GetMetrics(context.Context, *pb.GetDestinationMetrics_Request) (*pb.GetDestinationMetrics_Response, error) { |
| 156 | + stats := s.Plugin.Metrics() |
| 157 | + b, err := json.Marshal(stats) |
| 158 | + if err != nil { |
| 159 | + return nil, fmt.Errorf("failed to marshal stats: %w", err) |
| 160 | + } |
| 161 | + return &pb.GetDestinationMetrics_Response{ |
| 162 | + Metrics: b, |
| 163 | + }, nil |
| 164 | +} |
| 165 | + |
| 166 | +func (s *Server) DeleteStale(ctx context.Context, req *pb.DeleteStale_Request) (*pb.DeleteStale_Response, error) { |
| 167 | + schemas, err := schema.NewSchemasFromBytes(req.Tables) |
| 168 | + if err != nil { |
| 169 | + return nil, status.Errorf(codes.InvalidArgument, "failed to create schemas: %v", err) |
| 170 | + } |
| 171 | + tables, err := schema.NewTablesFromArrowSchemas(schemas) |
| 172 | + if err != nil { |
| 173 | + return nil, status.Errorf(codes.InvalidArgument, "failed to create tables: %v", err) |
| 174 | + } |
| 175 | + |
| 176 | + if err := s.Plugin.DeleteStale(ctx, tables, req.Source, req.Timestamp.AsTime()); err != nil { |
| 177 | + return nil, err |
| 178 | + } |
| 179 | + |
| 180 | + return &pb.DeleteStale_Response{}, nil |
| 181 | +} |
| 182 | + |
| 183 | +func (s *Server) setPKsForTables(tables schema.Tables) { |
| 184 | + if s.spec.PKMode == specs.PKModeCQID { |
| 185 | + setCQIDAsPrimaryKeysForTables(tables) |
| 186 | + } |
| 187 | +} |
| 188 | + |
| 189 | +func (s *Server) Close(ctx context.Context, _ *pb.Close_Request) (*pb.Close_Response, error) { |
| 190 | + return &pb.Close_Response{}, s.Plugin.Close(ctx) |
| 191 | +} |
0 commit comments