Skip to content

Commit a9a4401

Browse files
rohit-nayak-psrohit-nayak-ps
committed
sync
Signed-off-by: Rohit Nayak <[email protected]>
1 parent f1d393a commit a9a4401

File tree

14 files changed

+337
-55
lines changed

14 files changed

+337
-55
lines changed

‎commands/materialize.md

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
### Materialize
2+
3+
```
4+
Materialize <json_spec>
5+
```
6+
7+
Materialize is a low level API that allows for generalized materialization of tables. The target tables
8+
can be copies, aggregations or views. The target tables are kept in sync in near-realtime.
9+
10+
You can specify multiple tables to materialize using the json_spec parameter.
11+
12+
#### JSON spec details
13+
14+
* *workflow* name to refer to this materialization
15+
* *source_keyspace* keyspace containing the source table
16+
* *target_keyspace* keyspace to materialize to
17+
* *table_settings* list of materialized views and the associated query
18+
* *target_table* name of target table which should already exist
19+
* *source_expression* the materialization query: it can be a
20+
Note:
21+
22+
There are special commands to perform common materialization tasks and you should prefer them
23+
to using Materialize directly.
24+
* If you just want to copy tables to a different keyspace use MoveTables.
25+
* If you want to change sharding strategies use Reshard instead
26+
27+
Example:
28+
29+
Materialize '{"workflow": "product_sales", "source_keyspace": "commerce", "target_keyspace": "customer", "table_settings": [{"target_table": "sales_by_sku", "source_expression": "select sku, count(*), sum(price) from corder group by order_id"}]}'

‎examples/adopt.sh

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ echo ">>>>> Step 8. Run VDiff after inserting data into the customer keyspace"
4444
sleep 15
4545
mysql -h 127.0.0.1 -P 15306 -u msandbox --password=msandbox customer < sql/insert_commerce_data_after_switch.sql
4646

47-
#//BUG: tables customer/corder not found in schema
48-
#exit
49-
5047
vtctlclient VDiff commerce.commerce2customer_reverse
5148

5249
echo "Steps for rolling back"

‎examples/env.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,5 @@ alias vtctlclient="command vtctlclient -server localhost:15999 -log_dir ${VTDATA
3030
# Make sure aliases are expanded in non-interactive shell
3131
shopt -s expand_aliases
3232

33+
export MYSQL="command mysql -h 127.0.0.1 -P 15306"
34+
export LVTCTL="command vtctlclient -server localhost:15999 -log_dir ${VTDATAROOT}/tmp -alsologtostderr"

‎examples/load.sh

Lines changed: 160 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,163 @@
1-
echo ">>>>> Cleaning up previous runs, if any"
2-
./cleanup.sh
31

4-
source ./env.sh
2+
setup(){
3+
echo ">>>>> Cleaning up previous runs, if any"
4+
./cleanup.sh
5+
6+
7+
echo ">>>>>> Starting etcd"
8+
scripts/etcd-up.sh
9+
echo ">>>>>> Starting vtctld"
10+
scripts/vtctld-up.sh
11+
echo ">>>>>> Starting vtgate"
12+
scripts/vtgate-up.sh
13+
sleep 2
14+
mysql -h 127.0.0.1 -P 19327 -u msandbox --password=msandbox commerce -e "truncate table _vt.resharding_journal"
15+
mysql -h 127.0.0.1 -P 19327 -u msandbox --password=msandbox commerce -e "truncate table _vt.vreplication"
16+
}
17+
18+
unmanaged(){
19+
echo ">>>>>> Step 1: Starting unmanaged tablet (load1 keyspace)"
20+
scripts/unmanaged_load.sh
21+
sleep 5
22+
}
23+
24+
movetables(){
25+
echo ">>>>>> Step 2: Starting load2 tablets"
26+
for shard in "0"; do
27+
for i in 200 201 202; do
28+
CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
29+
CELL=zone1 KEYSPACE=load2 TABLET_UID=$i SHARD=$shard ./scripts/vttablet-up.sh
30+
done
31+
done
32+
33+
sleep 10
34+
35+
$LVTCTL InitShardMaster -force load2/0 zone1-200
36+
37+
sleep 2
38+
echo ">>>>>> Step 3. Calling MoveTables"
39+
TABLE=c2
40+
WORKFLOW=mt
41+
SOURCE_KS=load1
42+
TARGET_KS=load2
43+
KSWF=$TARGET_KS.$WORKFLOW
44+
$LVTCTL MoveTables -tablet_types=MASTER -workflow=$WORKFLOW $SOURCE_KS $TARGET_KS $TABLE
45+
if [ $? -eq 1 ]
46+
then
47+
echo "Error in MoveTables, exiting"
48+
exit
49+
fi
50+
51+
# TABLE2=c3
52+
# WORKFLOW2=mt2
53+
# $LVTCTL MoveTables -tablet_types=MASTER -workflow=$WORKFLOW2 $SOURCE_KS $TARGET_KS $TABLE2
54+
# if [ $? -eq 1 ]
55+
# then
56+
# echo "Error in MoveTables, exiting"
57+
# exit
58+
# fi
59+
exit
60+
echo ">>>>> Step 4. Waiting for Vreplication to copy the data ..."
61+
sleep 5
62+
echo ">>>>> Step 5. Run VDiff "
63+
$LVTCTL VDiff $KSWF
64+
65+
echo ">>>>>> Step 6. Calling SwitchReads"
66+
$LVTCTL SwitchReads -tablet_type=rdonly $KSWF
67+
if [ $? -eq 1 ]
68+
then
69+
echo "Error in SwitchReads, exiting"
70+
exit
71+
fi
72+
$LVTCTL SwitchReads -tablet_type=replica $KSWF
73+
sleep 5
74+
echo ">>>>>> Step 7. Calling SwitchWrites"
75+
$LVTCTL SwitchWrites $KSWF
76+
sleep 2
77+
#$LVTCTL DropSources $KSWF
78+
#TODO: if you remove this you get the "buildResharder: readRefStreams: blsIsReference: table c2 not found in vschema" ERROR
79+
/usr/bin/mysql -S /home/rohit/vtdataroot/vt_0000000200/mysql.sock -u vt_dba -e "delete from _vt.vreplication"
80+
}
81+
82+
reshard() {
83+
# Reshard
84+
WORKFLOW=rs
85+
TARGET_KS=load2
86+
KSWF="$TARGET_KS.$WORKFLOW"
587

6-
echo ">>>>>> Starting etcd"
7-
scripts/etcd-up.sh
8-
echo ">>>>>> Starting vtctld"
9-
scripts/vtctld-up.sh
10-
echo ">>>>>> Starting vtgate"
11-
scripts/vtgate-up.sh
12-
sleep 2
13-
echo ">>>>>> Step 1: Starting unmanaged tablet (load1 keyspace)"
14-
scripts/unmanaged_load.sh
15-
sleep 5
16-
17-
echo ">>>>>> Step 2: Starting load2 tablets"
18-
for shard in "0"; do
19-
for i in 200 201 202; do
20-
CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
21-
CELL=zone1 KEYSPACE=load2 TABLET_UID=$i SHARD=$shard ./scripts/vttablet-up.sh
88+
echo ">>>>>>>> applying schema/vschema"
89+
$LVTCTL ApplySchema -sql-file sql/c2_seq.sql $TARGET_KS
90+
$LVTCTL ApplyVSchema -vschema_file sql/load2_sharded_vschema.json $TARGET_KS
91+
$LVTCTL RebuildVSchemaGraph -cells=zone1
92+
93+
echo ">>>>>> Step 8: Starting $TARGET_KS tablets"
94+
shards=('-80' '80-')
95+
for ((idx=0; idx<${#shards[@]}; ++idx)); do
96+
shard=${shards[idx]}
97+
echo "idx $idx, shard is $shard"
98+
for i in 1 2 3; do
99+
TID=`expr 299 + $idx \* 100 + $i`
100+
echo "Creating tablet $TID"
101+
CELL=zone1 TABLET_UID=$TID ./scripts/mysqlctl-up.sh
102+
CELL=zone1 KEYSPACE=$TARGET_KS TABLET_UID=$TID SHARD="$shard" ./scripts/vttablet-up.sh
103+
done
22104
done
23-
done
24-
25-
sleep 10
26-
27-
vtctlclient -server localhost:15999 InitShardMaster -force load2/0 zone1-200
28-
sleep 2
29-
vtctlclient -server localhost:15999 MoveTables -tablet_types=MASTER -workflow=wf1 load1 load2 c1
30-
sleep 5
31-
vtctlclient -server localhost:15999 VDiff load2.wf1
32-
exit
33-
34-
sleep 2
35-
echo ">>>>>> Step 3. Calling MoveTables"
36-
37-
vtctlclient MoveTables -tablet_types=MASTER -workflow=wf1 load1 load2 c1
38-
#vtctlclient MoveTables -tablet_types=MASTER -workflow=wf2 load1 load2 table2
39-
#vtctlclient MoveTables -tablet_types=MASTER -workflow=wf3 load1 load2 table3
40-
#vtctlclient MoveTables -tablet_types=MASTER -workflow=wf4 load1 load2 table4
41-
42-
echo ">>>>> Step 4. Waiting for Vreplication to copy the data ..."
43-
sleep 5
44-
echo ">>>>> Step 5. Run VDiff "
45-
vtctlclient VDiff load2.wf
46-
echo ">>>>>> Step 6. Calling SwitchReads"
47-
vtctlclient SwitchReads -tablet_type=rdonly load2.wf
48-
vtctlclient SwitchReads -tablet_type=replica load2.wf
49-
sleep 5
50-
echo ">>>>>> Step 7. Calling SwitchWrites"
51-
vtctlclient SwitchWrites load2.wf
105+
106+
sleep 10
107+
108+
$LVTCTL InitShardMaster -force $TARGET_KS/-80 zone1-300
109+
$LVTCTL InitShardMaster -force $TARGET_KS/80- zone1-400
110+
sleep 2
111+
echo ">>>>>> Step 9. Calling Reshard"
112+
113+
$LVTCTL Reshard -tablet_types=MASTER $KSWF "0" "-80,80-"
114+
if [ $? -eq 1 ]
115+
then
116+
echo "Error in Reshard, exiting"
117+
exit
118+
fi
119+
echo ">>>>> Step 4. Waiting for Vreplication to copy the data ..."
120+
sleep 5
121+
echo ">>>>> Step 5. Run VDiff "
122+
$LVTCTL VDiff $KSWF
123+
124+
echo ">>>>>> Step 6. Calling SwitchReads"
125+
$LVTCTL SwitchReads -tablet_type=rdonly $KSWF
126+
$LVTCTL SwitchReads -tablet_type=replica $KSWF
127+
sleep 5
128+
echo ">>>>>> Step 7. Calling SwitchWrites"
129+
$LVTCTL SwitchWrites $KSWF
130+
}
131+
132+
133+
source ./env.sh
134+
135+
main() {
136+
source ./step.sh
137+
STEP="${STEP:=0}"
138+
NEXTSTEP="$(($STEP+1))"
139+
echo "export STEP='$NEXTSTEP'" > step.sh
140+
141+
case "$NEXTSTEP" in
142+
143+
"1") setup
144+
unmanaged
145+
movetables
146+
;;
147+
"2") reshard
148+
;;
149+
*) echo "Unknown parameter"
150+
;;
151+
esac
152+
}
153+
154+
mt() {
155+
setup
156+
unmanaged
157+
movetables
158+
}
159+
rs() {
160+
reshard
161+
}
162+
mt
163+
#rs

‎examples/scripts/pop

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
go run populate2.go

‎examples/scripts/populate.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"time"
6+
"database/sql"
7+
_ "github.com/go-sql-driver/mysql"
8+
"github.com/paulbellamy/ratecounter"
9+
)
10+
11+
func main() {
12+
// Open up our database connection.
13+
// I've set up a database on my local machine using phpmyadmin.
14+
// The database is called testDb
15+
16+
17+
// perform a db.Query insert
18+
counter := ratecounter.NewRateCounter(1 * time.Second)
19+
done := false
20+
var i int64
21+
i = 0
22+
db, err := sql.Open("mysql", "msandbox:msandbox@tcp(127.0.0.1:19327)/test")
23+
fmt.Println("connection opened")
24+
if err != nil {
25+
panic(err.Error())
26+
}
27+
defer db.Close()
28+
now := time.Now().UnixNano()
29+
db.Query("BEGIN")
30+
for !done {
31+
32+
i++
33+
// if there is an error opening the connection, handle it
34+
rows, err := db.Query(fmt.Sprintf("INSERT INTO c1 VALUES ( %d, 'TEST' )", now+i))
35+
if err != nil {
36+
panic(err.Error())
37+
}
38+
if rows != nil {
39+
rows.Close()
40+
}
41+
counter.Incr(1)
42+
if i % 1000 == 0 {
43+
db.Query("COMMIT")
44+
fmt.Printf("QPS: %d\n", counter.Rate())
45+
db.Query("BEGIN")
46+
/*
47+
rows, err := db.Query(fmt.Sprintf("update c1 set val2 = 'x%d' where c1 < 1000", now))
48+
if err != nil {
49+
panic(err.Error())
50+
}
51+
52+
if rows != nil {
53+
rows.Close()
54+
}
55+
*/
56+
57+
}
58+
}
59+
60+
}

‎examples/scripts/populate2.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package main
2+
import (
3+
"fmt"
4+
"time"
5+
"database/sql"
6+
_ "github.com/go-sql-driver/mysql"
7+
"github.com/paulbellamy/ratecounter"
8+
)
9+
10+
func PrepareTx(db *sql.DB,qry string) (tx *sql.Tx, s *sql.Stmt, e error) {
11+
if tx,e=db.Begin(); e!=nil {
12+
return
13+
}
14+
15+
if s, e = tx.Prepare(qry);e!=nil {
16+
panic(e.Error())
17+
}
18+
return
19+
}
20+
21+
func main() {
22+
counter := ratecounter.NewRateCounter(1 * time.Second)
23+
done := false
24+
var i int64
25+
query := "insert into c1(c1) values (?)"
26+
i = 0
27+
db, err := sql.Open("mysql", "msandbox:msandbox@tcp(127.0.0.1:19327)/test")
28+
fmt.Println("connection opened")
29+
if err != nil {
30+
panic(err.Error())
31+
}
32+
defer db.Close()
33+
now := time.Now().UnixNano()
34+
tx, stmt, err := PrepareTx(db, query)
35+
if err != nil {
36+
panic(err.Error())
37+
}
38+
for !done {
39+
//time.Sleep(1 * time.Millisecond)
40+
i++
41+
counter.Incr(1)
42+
if _,err := stmt.Exec(now+i); err != nil {
43+
panic(err)
44+
}
45+
if i % 10000 == 0 {
46+
fmt.Printf("QPS: %d\n", counter.Rate())
47+
if err := tx.Commit(); err!=nil {
48+
panic(err)
49+
}
50+
tx, stmt, err = PrepareTx(db, query)
51+
if err != nil {
52+
panic(err.Error())
53+
}
54+
}
55+
}
56+
57+
}

‎examples/scripts/unmanaged_load.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ vttablet \
3030
-db_filtered_password $DB_PASS \
3131
-db_repl_user $DB_USER \
3232
-db_repl_password $DB_PASS \
33+
-track_schema_versions=true \
3334
-vtctld_addr http://$hostname:$vtctld_web_port/ \
3435
-init_db_name_override test \
3536
-init_populate_metadata \

‎examples/scripts/vttablet-up.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ vttablet \
4646
-init_tablet_type $tablet_type \
4747
-health_check_interval 5s \
4848
-enable_semi_sync \
49-
-watch_replication_stream \
5049
-enable_replication_reporter \
5150
-backup_storage_implementation file \
5251
-file_backup_storage_root $VTDATAROOT/backups \

‎examples/sql/c2_seq.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
create table c2_seq(id int, next_id bigint, cache bigint, primary key(id)) comment 'vitess_sequence';
2+
insert into c2_seq(id, next_id, cache) values(0, 1000, 100);

0 commit comments

Comments
 (0)