Skip to content

Commit 68242a6

Browse files
beingnoble03beingnoble03
authored
LookupVindex: Multiple lookup tables support for LookupVindexCreate (vitessio#17566)
Signed-off-by: Noble Mittal <noblemittal@outlook.com>
1 parent f0e9a2d commit 68242a6

File tree

11 files changed

+3379
-2613
lines changed

11 files changed

+3379
-2613
lines changed

‎go/cmd/vtctldclient/command/vreplication/lookupvindex/lookupvindex.go‎

Lines changed: 126 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ limitations under the License.
1717
package lookupvindex
1818

1919
import (
20+
"encoding/json"
2021
"fmt"
22+
"os"
2123
"strings"
2224

2325
"github.com/spf13/cobra"
@@ -31,6 +33,16 @@ import (
3133
topoprotopb "vitess.io/vitess/go/vt/topo/topoproto"
3234
)
3335

36+
// vindexParams is used to unmarshal content from params-file.
37+
type vindexParams struct {
38+
LookupVindexType string `json:"lookup_vindex_type"`
39+
TableOwner string `json:"table_owner"`
40+
TableOwnerColumns []string `json:"table_owner_columns"`
41+
TableName string `json:"table_name"`
42+
TableVindexType string `json:"table_vindex_type"`
43+
IgnoreNulls bool `json:"ignore_nulls"`
44+
}
45+
3446
var (
3547
tabletTypesDefault = []topodatapb.TabletType{
3648
topodatapb.TabletType_REPLICA,
@@ -68,6 +80,7 @@ var (
6880
TabletTypesInPreferenceOrder bool
6981
IgnoreNulls bool
7082
ContinueAfterCopyWithOwner bool
83+
ParamsFile string
7184
}{}
7285

7386
externalizeOptions = struct {
@@ -84,6 +97,36 @@ var (
8497
}{}
8598

8699
parseAndValidateCreate = func(cmd *cobra.Command, args []string) error {
100+
if createOptions.ParamsFile != "" {
101+
if createOptions.TableOwner != "" {
102+
return fmt.Errorf("cannot specify both table-owner and params-file")
103+
}
104+
if createOptions.Type != "" {
105+
return fmt.Errorf("cannot specify both type and params-file")
106+
}
107+
if len(createOptions.TableOwnerColumns) != 0 {
108+
return fmt.Errorf("cannot specify both table-owner-columns and params-file")
109+
}
110+
paramsFile, err := os.ReadFile(createOptions.ParamsFile)
111+
if err != nil {
112+
return err
113+
}
114+
createVindexParams := map[string]*vindexParams{}
115+
err = json.Unmarshal(paramsFile, &createVindexParams)
116+
if err != nil {
117+
return err
118+
}
119+
return parseVindexParams(createVindexParams, cmd)
120+
}
121+
if createOptions.TableOwner == "" {
122+
return fmt.Errorf("table-owner is a required flag")
123+
}
124+
if createOptions.Type == "" {
125+
return fmt.Errorf("type is a required flag")
126+
}
127+
if len(createOptions.TableOwnerColumns) == 0 {
128+
return fmt.Errorf("table-owner-columns is a required flag")
129+
}
87130
if createOptions.TableName == "" { // Use vindex name
88131
createOptions.TableName = baseOptions.Name
89132
}
@@ -139,6 +182,86 @@ var (
139182
return nil
140183
}
141184

185+
parseVindexParams = func(params map[string]*vindexParams, cmd *cobra.Command) error {
186+
if len(params) == 0 {
187+
return fmt.Errorf("at least 1 vindex is required")
188+
}
189+
190+
vindexes := map[string]*vschemapb.Vindex{}
191+
tables := map[string]*vschemapb.Table{}
192+
for vindexName, vindex := range params {
193+
if len(vindex.TableOwnerColumns) == 0 {
194+
return fmt.Errorf("table owner columns found empty for '%s'", vindexName)
195+
}
196+
if vindex.TableOwner == "" {
197+
return fmt.Errorf("table owner found empty for '%s'", vindexName)
198+
}
199+
if vindex.TableName == "" {
200+
vindex.TableName = vindexName
201+
}
202+
203+
if !strings.Contains(vindex.LookupVindexType, "lookup") {
204+
return fmt.Errorf("%s is not a lookup vindex type", vindex.LookupVindexType)
205+
}
206+
207+
vindexes[vindexName] = &vschemapb.Vindex{
208+
Type: vindex.LookupVindexType,
209+
Params: map[string]string{
210+
"table": baseOptions.TableKeyspace + "." + vindex.TableName,
211+
"from": strings.Join(vindex.TableOwnerColumns, ","),
212+
"to": "keyspace_id",
213+
"ignore_nulls": fmt.Sprintf("%t", vindex.IgnoreNulls),
214+
},
215+
Owner: vindex.TableOwner,
216+
}
217+
218+
targetTableColumnVindex := &vschemapb.ColumnVindex{
219+
// If the vindex type is empty then we'll fill this later by
220+
// choosing the most appropriate vindex type for the given column.
221+
Name: vindex.TableVindexType,
222+
Columns: vindex.TableOwnerColumns,
223+
}
224+
sourceTableColumnVindex := &vschemapb.ColumnVindex{
225+
Name: vindexName,
226+
Columns: vindex.TableOwnerColumns,
227+
}
228+
229+
if table, ok := tables[vindex.TableName]; !ok {
230+
tables[vindex.TableName] = &vschemapb.Table{
231+
ColumnVindexes: []*vschemapb.ColumnVindex{targetTableColumnVindex},
232+
}
233+
} else {
234+
table.ColumnVindexes = append(table.ColumnVindexes, targetTableColumnVindex)
235+
}
236+
237+
if table, ok := tables[vindex.TableOwner]; !ok {
238+
tables[vindex.TableOwner] = &vschemapb.Table{
239+
ColumnVindexes: []*vschemapb.ColumnVindex{sourceTableColumnVindex},
240+
}
241+
} else {
242+
table.ColumnVindexes = append(table.ColumnVindexes, sourceTableColumnVindex)
243+
}
244+
}
245+
246+
baseOptions.Vschema = &vschemapb.Keyspace{
247+
Vindexes: vindexes,
248+
Tables: tables,
249+
}
250+
251+
// VReplication specific flags.
252+
ttFlag := cmd.Flags().Lookup("tablet-types")
253+
if ttFlag != nil && ttFlag.Changed {
254+
createOptions.TabletTypes = tabletTypesDefault
255+
}
256+
cFlag := cmd.Flags().Lookup("cells")
257+
if cFlag != nil && cFlag.Changed {
258+
for i, cell := range createOptions.Cells {
259+
createOptions.Cells[i] = strings.TrimSpace(cell)
260+
}
261+
}
262+
return nil
263+
}
264+
142265
// cancel makes a WorkflowDelete call to a vtctld.
143266
cancel = &cobra.Command{
144267
Use: "cancel",
@@ -166,7 +289,7 @@ var (
166289
// create makes a LookupVindexCreate call to a vtctld.
167290
create = &cobra.Command{
168291
Use: "create",
169-
Short: "Create the Lookup Vindex in the specified keyspace and backfill it with a VReplication workflow.",
292+
Short: "Create the Lookup Vindex(es) in the specified keyspace and backfill them with a VReplication workflow.",
170293
Example: `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer create --keyspace customer --type consistent_lookup_unique --table-owner corder --table-owner-columns sku --table-name corder_lookup_tbl --table-vindex-type unicode_loose_xxhash`,
171294
SilenceUsage: true,
172295
DisableFlagsInUseLine: true,
@@ -359,7 +482,7 @@ func commandShow(cmd *cobra.Command, args []string) error {
359482
}
360483

361484
func registerCommands(root *cobra.Command) {
362-
base.PersistentFlags().StringVar(&baseOptions.Name, "name", "", "The name of the Lookup Vindex to create. This will also be the name of the VReplication workflow created to backfill the Lookup Vindex.")
485+
base.PersistentFlags().StringVar(&baseOptions.Name, "name", "", "The name of the Lookup Vindex to create. This will also be the name of the VReplication workflow created to backfill the Lookup Vindex. This will be used only for the workflow name if params-file is used.")
363486
base.MarkPersistentFlagRequired("name")
364487
base.PersistentFlags().StringVar(&baseOptions.TableKeyspace, "table-keyspace", "", "The keyspace to create the lookup table in. This is also where the VReplication workflow is created to backfill the Lookup Vindex.")
365488
base.MarkPersistentFlagRequired("table-keyspace")
@@ -370,15 +493,13 @@ func registerCommands(root *cobra.Command) {
370493
create.Flags().StringVar(&createOptions.Keyspace, "keyspace", "", "The keyspace to create the Lookup Vindex in. This is also where the table-owner must exist.")
371494
create.MarkFlagRequired("keyspace")
372495
create.Flags().StringVar(&createOptions.Type, "type", "", "The type of Lookup Vindex to create.")
373-
create.MarkFlagRequired("type")
374496
create.Flags().StringVar(&createOptions.TableOwner, "table-owner", "", "The table holding the data which we should use to backfill the Lookup Vindex. This must exist in the same keyspace as the Lookup Vindex.")
375-
create.MarkFlagRequired("table-owner")
376497
create.Flags().StringSliceVar(&createOptions.TableOwnerColumns, "table-owner-columns", nil, "The columns to read from the owner table. These will be used to build the hash which gets stored as the keyspace_id value in the lookup table.")
377-
create.MarkFlagRequired("table-owner-columns")
378498
create.Flags().StringVar(&createOptions.TableName, "table-name", "", "The name of the lookup table. If not specified, then it will be created using the same name as the Lookup Vindex.")
379499
create.Flags().StringVar(&createOptions.TableVindexType, "table-vindex-type", "", "The primary vindex name/type to use for the lookup table, if the table-keyspace is sharded. If no value is provided then the default type will be used based on the table-owner-columns types.")
380500
create.Flags().BoolVar(&createOptions.IgnoreNulls, "ignore-nulls", false, "Do not add corresponding records in the lookup table if any of the owner table's 'from' fields are NULL.")
381501
create.Flags().BoolVar(&createOptions.ContinueAfterCopyWithOwner, "continue-after-copy-with-owner", true, "Vindex will continue materialization after the backfill completes when an owner is provided.")
502+
create.Flags().StringVar(&createOptions.ParamsFile, "params-file", "", "JSON file containing lookup vindex parameters. Use this for creating multiple lookup vindexes.")
382503
// VReplication specific flags.
383504
create.Flags().StringSliceVar(&createOptions.Cells, "cells", nil, "Cells to look in for source tablets to replicate from.")
384505
create.Flags().Var((*topoprotopb.TabletTypeListFlag)(&createOptions.TabletTypes), "tablet-types", "Source tablet types to replicate from.")

0 commit comments

Comments
 (0)