Skip to content

Commit 4259dab

Browse files
yinhaofengyinhaofeng
committed
milvus
1 parent 36839eb commit 4259dab

File tree

3 files changed

+210
-0
lines changed

3 files changed

+210
-0
lines changed

‎models/demo/movie_recommand/get_movie_vectors.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""
15+
feed_var {
16+
name: "movieid"
17+
alias_name: "movieid"
18+
is_lod_tensor: true
19+
feed_type: 0
20+
shape: -1
21+
}
22+
feed_var {
23+
name: "title"
24+
alias_name: "title"
25+
is_lod_tensor: true
26+
feed_type: 0
27+
shape: -1
28+
}
29+
feed_var {
30+
name: "genres"
31+
alias_name: "genres"
32+
is_lod_tensor: true
33+
feed_type: 0
34+
shape: -1
35+
}
36+
fetch_var {
37+
name: "save_infer_model/scale_0.tmp_0"
38+
alias_name: "save_infer_model/scale_0.tmp_0"
39+
is_lod_tensor: false
40+
fetch_type: 1
41+
shape: 32
42+
}
43+
"""
44+
45+
from paddle_serving_app.local_predict import LocalPredictor
46+
import redis
47+
import numpy as np
48+
import codecs
49+
50+
51+
class Movie(object):
52+
def __init__(self):
53+
self.movie_id, self.title, self.genres = "", "", ""
54+
pass
55+
56+
57+
def hash2(a):
58+
return hash(a) % 600000
59+
60+
61+
ctr_client = LocalPredictor()
62+
ctr_client.load_model_config("serving_server")
63+
with codecs.open("movies.dat", "r", encoding='utf-8', errors='ignore') as f:
64+
lines = f.readlines()
65+
66+
ff = open("movie_vectors.txt", 'w')
67+
68+
for line in lines:
69+
if len(line.strip()) == 0:
70+
continue
71+
tmp = line.strip().split("::")
72+
movie_id = tmp[0]
73+
title = tmp[1]
74+
genre_group = tmp[2]
75+
76+
tmp = genre_group.strip().split("|")
77+
genre = tmp
78+
movie = Movie()
79+
item_infos = []
80+
if isinstance(genre, list):
81+
movie.genres = genre
82+
else:
83+
movie.genres = [genre]
84+
movie.movie_id, movie.title = movie_id, title
85+
item_infos.append(movie)
86+
87+
dic = {"movieid": [], "title": [], "genres": []}
88+
batch_size = len(item_infos)
89+
for i, item_info in enumerate(item_infos):
90+
dic["movieid"].append(hash2(item_info.movie_id))
91+
dic["title"].append(hash2(item_info.title))
92+
dic["genres"].extend([hash2(x) for x in item_info.genres])
93+
94+
dic["title"][:4]
95+
if len(dic["title"]) <= 4:
96+
for i in range(4 - len(dic["title"])):
97+
dic["title"].append("0")
98+
dic["genres"][:3]
99+
if len(dic["genres"]) <= 3:
100+
for i in range(3 - len(dic["genres"])):
101+
dic["genres"].append("0")
102+
print(dic["movieid"], dic["title"], dic["genres"])
103+
dic["movieid"] = np.array(dic["movieid"]).astype(np.int64).reshape(-1, 1)
104+
dic["title"] = np.array(dic["title"]).astype(np.int64).reshape(-1, 4)
105+
dic["genres"] = np.array(dic["genres"]).astype(np.int64).reshape(-1, 3)
106+
107+
fetch_map = ctr_client.predict(
108+
feed=dic, fetch=["save_infer_model/scale_0.tmp_0"], batch=True)
109+
ff.write("{}:{}\n".format(movie_id,
110+
str(fetch_map["save_infer_model/scale_0.tmp_0"]
111+
.tolist()[0])))
112+
ff.close()

‎models/demo/movie_recommand/recall/movie.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
runner:
16+
train_data_dir: "../data/train"
17+
train_reader_path: "reader" # importlib format
18+
train_batch_size: 1
19+
model_save_path: "movie_model"
20+
21+
use_gpu: True
22+
epochs: 5
23+
print_interval: 20
24+
25+
test_data_dir: "../data/test"
26+
infer_reader_path: "reader" # importlib format
27+
infer_batch_size: 1
28+
infer_load_path: "movie_model"
29+
infer_start_epoch: 4
30+
infer_end_epoch: 5
31+
32+
runner_result_dump_path: "recall_infer_result"
33+
34+
#use inference save model
35+
use_inference: True
36+
save_inference_feed_varnames: ["movieid", "title", "genres"]
37+
save_inference_fetch_varnames: ["linear_15.tmp_1"]
38+
39+
# hyper parameters of user-defined network
40+
hyper_parameters:
41+
# optimizer config
42+
optimizer:
43+
class: Adam
44+
learning_rate: 0.001
45+
# user-defined <key, value> pairs
46+
sparse_feature_number: 600000
47+
sparse_feature_dim: 9
48+
dense_input_dim: 13
49+
fc_sizes: [512, 256, 128, 32]

‎models/demo/movie_recommand/recall/user.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
runner:
16+
train_data_dir: "../data/train"
17+
train_reader_path: "reader" # importlib format
18+
train_batch_size: 128
19+
model_save_path: "user_model"
20+
21+
use_gpu: True
22+
epochs: 5
23+
print_interval: 20
24+
25+
test_data_dir: "../data/test"
26+
infer_reader_path: "reader" # importlib format
27+
infer_batch_size: 128
28+
infer_load_path: "user_model"
29+
infer_start_epoch: 4
30+
infer_end_epoch: 5
31+
32+
runner_result_dump_path: "recall_infer_result"
33+
34+
#use inference save model
35+
use_inference: True
36+
save_inference_feed_varnames: ["userid", "gender", "age", "occupation"]
37+
save_inference_fetch_varnames: ["linear_11.tmp_1"]
38+
39+
# hyper parameters of user-defined network
40+
hyper_parameters:
41+
# optimizer config
42+
optimizer:
43+
class: Adam
44+
learning_rate: 0.001
45+
# user-defined <key, value> pairs
46+
sparse_feature_number: 600000
47+
sparse_feature_dim: 9
48+
dense_input_dim: 13
49+
fc_sizes: [512, 256, 128, 32]

0 commit comments

Comments
 (0)