Skip to content

Commit f0e76d1

Browse files
wanyaoqiwanyaoqi
authored
fix(host): persistent ceph config for qemu connection (#23088)
1 parent a2ba9e5 commit f0e76d1

File tree

5 files changed

+114
-36
lines changed

5 files changed

+114
-36
lines changed

‎cmd/climc/shell/compute/storages.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,7 @@ func init() {
7474
if info.StorageType != "rbd" {
7575
return errors.Errorf("invalid storage_type %s", info.StorageType)
7676
}
77-
cli, err := cephutils.NewClient(
78-
info.StorageConf.MonHost,
79-
info.StorageConf.Key,
80-
info.StorageConf.Pool,
81-
info.StorageConf.EnableMessengerV2,
82-
)
77+
cli, err := cephutils.NewClient(info.StorageConf.MonHost, info.StorageConf.Key, info.StorageConf.Pool, info.StorageConf.EnableMessengerV2, 0, 0, 0)
8378
if err != nil {
8479
return errors.Wrap(err, "cephutils.NewClient")
8580
}

‎pkg/hostman/storageman/storage_rbd.go‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"context"
1919
"fmt"
2020
"os"
21+
"path"
2122
"path/filepath"
2223

2324
"yunion.io/x/jsonutils"
@@ -54,9 +55,9 @@ type sStorageConf struct {
5455
// The messenger v2 protocol, or msgr2, is the second major
5556
// revision on Ceph’s on-wire protocol.
5657
EnableMessengerV2 bool
57-
RadosMonOpTimeout int64
58-
RadosOsdOpTimeout int64
59-
ClientMountTimeout int64
58+
RadosMonOpTimeout int
59+
RadosOsdOpTimeout int
60+
ClientMountTimeout int
6061
}
6162

6263
type SRbdStorage struct {
@@ -102,7 +103,7 @@ func (s *SRbdStorage) getCephClient(pool string) (*cephutils.CephClient, error)
102103
if pool == "" {
103104
pool = s.Pool
104105
}
105-
return cephutils.NewClient(s.MonHost, s.Key, pool, s.EnableMessengerV2)
106+
return cephutils.NewClient(s.MonHost, s.Key, pool, s.EnableMessengerV2, s.RadosMonOpTimeout, s.RadosOsdOpTimeout, s.ClientMountTimeout)
106107
}
107108

108109
func (s *SRbdStorage) getClient() (*cephutils.CephClient, error) {
@@ -139,9 +140,17 @@ func (s *SRbdStorage) GetImgsaveBackupPath() string {
139140
return ""
140141
}
141142

143+
func (s *SRbdStorage) getStorageConfDir() string {
144+
workspacePath := filepath.Dir(options.HostOptions.ServersPath)
145+
cephConfDir := path.Join(workspacePath, "ceph_config", s.StorageId)
146+
return cephConfDir
147+
}
148+
142149
// Tip Configuration values containing :, @, or = can be escaped with a leading \ character.
143150
func (s *SRbdStorage) getStorageConfString() string {
144-
return cephutils.CephConfString(s.MonHost, s.Key, s.RadosMonOpTimeout, s.RadosOsdOpTimeout, s.ClientMountTimeout)
151+
cephConfDir := s.getStorageConfDir()
152+
cephConfPath := path.Join(cephConfDir, "ceph.conf")
153+
return ":" + fmt.Sprintf("conf=%s", cephConfPath)
145154
}
146155

147156
func (s *SRbdStorage) listImages() ([]string, error) {
@@ -697,6 +706,25 @@ func (s *SRbdStorage) SetStorageInfo(storageId, storageName string, conf jsonuti
697706
if s.ClientMountTimeout == 0 {
698707
s.ClientMountTimeout = api.RBD_DEFAULT_MOUNT_TIMEOUT
699708
}
709+
confDir := s.getStorageConfDir()
710+
if err := os.MkdirAll(confDir, os.ModePerm); err != nil {
711+
return errors.Wrapf(err, "mkdir all %s", confDir)
712+
}
713+
confPath := path.Join(confDir, "ceph.conf")
714+
keyringPath := path.Join(confDir, "ceph.keyring")
715+
716+
cli, err := cephutils.NewClientAndPersistentConf(
717+
s.MonHost, s.Key, s.Pool,
718+
s.EnableMessengerV2, s.RadosMonOpTimeout,
719+
s.RadosOsdOpTimeout, s.ClientMountTimeout,
720+
confPath, keyringPath,
721+
)
722+
if err != nil {
723+
log.Errorf("failed get ceph client %s:", err)
724+
return errors.Wrap(err, "getClient")
725+
}
726+
defer cli.Close()
727+
700728
return nil
701729
}
702730

‎pkg/image/models/image_ceph_cache_storages.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func getCephStorages() *SCephStorageConf {
114114
}
115115

116116
func getStorageFsid(conf *computeapi.RbdStorageConf) string {
117-
cli, err := cephutils.NewClient(conf.MonHost, conf.Key, conf.Pool, conf.EnableMessengerV2)
117+
cli, err := cephutils.NewClient(conf.MonHost, conf.Key, conf.Pool, conf.EnableMessengerV2, 0, 0, 0)
118118
if err != nil {
119119
log.Errorf("failed new client of ceph storage %s:%s", conf.MonHost, conf.Pool)
120120
return ""

‎pkg/image/models/images.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2192,9 +2192,9 @@ func (img *SImage) cacheToCephStorages(ctx context.Context) {
21922192
storageConfString := cephutils.CephConfString(
21932193
storageConf.MonHost,
21942194
storageConf.Key,
2195-
int64(storageConf.RadosMonOpTimeout),
2196-
int64(storageConf.RadosOsdOpTimeout),
2197-
int64(storageConf.ClientMountTimeout),
2195+
storageConf.RadosMonOpTimeout,
2196+
storageConf.RadosOsdOpTimeout,
2197+
storageConf.ClientMountTimeout,
21982198
)
21992199
rbdPath := fmt.Sprintf("rbd:%s/%s%s", storageConf.Pool, imgTmpName, storageConfString)
22002200
log.Infof("convert local image %s to rbd pool %s", img.Id, storageConf.Pool)
@@ -2245,7 +2245,7 @@ func (img *SImage) cacheToCephStorages(ctx context.Context) {
22452245
}
22462246

22472247
func (img *SImage) removeCephImage(storageConf *computeapi.RbdStorageConf, imgName string) error {
2248-
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2)
2248+
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2, 0, 0, 0)
22492249
if err != nil {
22502250
return errors.Wrap(err, "cephutils.NewClient")
22512251
}
@@ -2261,7 +2261,7 @@ func (img *SImage) removeCephImage(storageConf *computeapi.RbdStorageConf, imgNa
22612261
}
22622262

22632263
func (img *SImage) renameCephImage(storageConf *computeapi.RbdStorageConf, srcImgName, destImgName string) error {
2264-
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2)
2264+
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2, 0, 0, 0)
22652265
if err != nil {
22662266
return errors.Wrap(err, "cephutils.NewClient")
22672267
}
@@ -2277,7 +2277,7 @@ func (img *SImage) getCephImage(storageConf *computeapi.RbdStorageConf, imgName
22772277
if imgName == "" {
22782278
imgName = "image_cache_" + img.Id
22792279
}
2280-
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2)
2280+
cli, err := cephutils.NewClient(storageConf.MonHost, storageConf.Key, storageConf.Pool, storageConf.EnableMessengerV2, 0, 0, 0)
22812281
if err != nil {
22822282
return nil, errors.Wrap(err, "cephutils.NewClient")
22832283
}
@@ -2292,9 +2292,9 @@ func (img *SImage) getCephImage(storageConf *computeapi.RbdStorageConf, imgName
22922292
storageConfString := cephutils.CephConfString(
22932293
storageConf.MonHost,
22942294
storageConf.Key,
2295-
int64(storageConf.RadosMonOpTimeout),
2296-
int64(storageConf.RadosOsdOpTimeout),
2297-
int64(storageConf.ClientMountTimeout),
2295+
storageConf.RadosMonOpTimeout,
2296+
storageConf.RadosOsdOpTimeout,
2297+
storageConf.ClientMountTimeout,
22982298
)
22992299
rbdPath := fmt.Sprintf("rbd:%s/%s%s", storageConf.Pool, imgName, storageConfString)
23002300
origin, err := qemuimg.NewQemuImage(rbdPath)
@@ -2309,7 +2309,7 @@ func (img *SImage) getCephImage(storageConf *computeapi.RbdStorageConf, imgName
23092309

23102310
func (img *SImage) cloneToCephStorage(ctx context.Context, monHost, key, pool string, enableMessengerV2 bool, destPool string) error {
23112311
imgName := "image_cache_" + img.Id
2312-
cli, err := cephutils.NewClient(monHost, key, pool, enableMessengerV2)
2312+
cli, err := cephutils.NewClient(monHost, key, pool, enableMessengerV2, 0, 0, 0)
23132313
if err != nil {
23142314
return errors.Wrap(err, "cephutils.NewClient")
23152315
}

‎pkg/util/cephutils/ceph.go‎

Lines changed: 69 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"yunion.io/x/pkg/errors"
2929
"yunion.io/x/pkg/utils"
3030

31+
api "yunion.io/x/onecloud/pkg/apis/compute"
3132
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
3233
"yunion.io/x/onecloud/pkg/util/fileutils2"
3334
"yunion.io/x/onecloud/pkg/util/procutils"
@@ -40,10 +41,14 @@ type CephClient struct {
4041
cephConf string
4142
keyConf string
4243

43-
timeout int
44+
timeout int
45+
persistentConf bool
4446
}
4547

4648
func (cli *CephClient) Close() error {
49+
if cli.persistentConf {
50+
return nil
51+
}
4752
if len(cli.keyConf) > 0 {
4853
os.Remove(cli.keyConf)
4954
}
@@ -236,7 +241,29 @@ func SetCephConfTempDir(dir string) {
236241
cephConfTmpDir = dir
237242
}
238243

239-
func NewClient(monHost, key, pool string, enableMessengerV2 bool) (*CephClient, error) {
244+
func NewClientAndPersistentConf(
245+
monHost, key, pool string, enableMessengerV2 bool,
246+
radosMonTimeout, radosOsdTimeout, clientMountTimeout int,
247+
confPath, keyringPath string,
248+
) (*CephClient, error) {
249+
if len(confPath) == 0 || len(keyringPath) == 0 {
250+
return nil, errors.Errorf("empty conf path %s %s", confPath, keyringPath)
251+
}
252+
return newClient(monHost, key, pool, enableMessengerV2, radosMonTimeout, radosOsdTimeout, clientMountTimeout, confPath, keyringPath)
253+
}
254+
255+
func NewClient(
256+
monHost, key, pool string, enableMessengerV2 bool,
257+
radosMonTimeout, radosOsdTimeout, clientMountTimeout int,
258+
) (*CephClient, error) {
259+
return newClient(monHost, key, pool, enableMessengerV2, radosMonTimeout, radosOsdTimeout, clientMountTimeout, "", "")
260+
}
261+
262+
func newClient(
263+
monHost, key, pool string, enableMessengerV2 bool,
264+
radosMonTimeout, radosOsdTimeout, clientMountTimeout int,
265+
confPath, keyringPath string,
266+
) (*CephClient, error) {
240267
client := &CephClient{
241268
monHost: monHost,
242269
key: key,
@@ -248,9 +275,18 @@ func NewClient(monHost, key, pool string, enableMessengerV2 bool) (*CephClient,
248275
keyring := fmt.Sprintf(`[client.admin]
249276
key = %s
250277
`, client.key)
251-
client.keyConf, err = writeFile("ceph.*.keyring", keyring)
252-
if err != nil {
253-
return nil, errors.Wrapf(err, "write keyring")
278+
if len(keyringPath) == 0 {
279+
client.keyConf, err = writeFile("ceph.*.keyring", keyring)
280+
if err != nil {
281+
return nil, errors.Wrapf(err, "write keyring")
282+
}
283+
} else {
284+
err = fileutils2.FilePutContents(keyringPath, keyring, false)
285+
if err != nil {
286+
return nil, errors.Wrapf(err, "write keyring to %s", keyringPath)
287+
}
288+
client.keyConf = keyringPath
289+
client.persistentConf = true
254290
}
255291
}
256292
monHosts := []string{}
@@ -264,13 +300,22 @@ func NewClient(monHost, key, pool string, enableMessengerV2 bool) (*CephClient,
264300
}
265301
}
266302
client.monHost = strings.Join(monHosts, ",")
303+
if radosMonTimeout <= 0 {
304+
radosMonTimeout = api.RBD_DEFAULT_MON_TIMEOUT
305+
}
306+
if radosOsdTimeout <= 0 {
307+
radosOsdTimeout = api.RBD_DEFAULT_OSD_TIMEOUT
308+
}
309+
if clientMountTimeout <= 0 {
310+
clientMountTimeout = api.RBD_DEFAULT_MOUNT_TIMEOUT
311+
}
267312

268313
conf := fmt.Sprintf(`[global]
269314
mon host = %s
270-
rados mon op timeout = 5
271-
rados osd_op timeout = 1200
272-
client mount timeout = 120
273-
`, client.monHost)
315+
rados mon op timeout = %d
316+
rados osd_op timeout = %d
317+
client mount timeout = %d
318+
`, client.monHost, radosMonTimeout, radosOsdTimeout, clientMountTimeout)
274319
if len(client.key) == 0 {
275320
conf = fmt.Sprintf(`%s
276321
auth_cluster_required = none
@@ -282,14 +327,24 @@ auth_client_required = none
282327
keyring = %s
283328
`, conf, client.keyConf)
284329
}
285-
client.cephConf, err = writeFile("ceph.*.conf", conf)
286-
if err != nil {
287-
return nil, errors.Wrapf(err, "write file")
330+
if len(confPath) == 0 {
331+
client.cephConf, err = writeFile("ceph.*.conf", conf)
332+
if err != nil {
333+
return nil, errors.Wrapf(err, "write file")
334+
}
335+
} else {
336+
err = fileutils2.FilePutContents(confPath, conf, false)
337+
if err != nil {
338+
return nil, errors.Wrapf(err, "write conf to %s", confPath)
339+
}
340+
client.cephConf = confPath
341+
client.persistentConf = true
288342
}
343+
289344
return client, nil
290345
}
291346

292-
func CephConfString(monHost, key string, radosMonOpTimeout, radosOsdOpTimeout, clientMountTimeout int64) string {
347+
func CephConfString(monHost, key string, radosMonOpTimeout, radosOsdOpTimeout, clientMountTimeout int) string {
293348
conf := []string{}
294349
monHosts := strings.Split(monHost, ",")
295350
for i := range monHosts {
@@ -305,7 +360,7 @@ func CephConfString(monHost, key string, radosMonOpTimeout, radosOsdOpTimeout, c
305360
}
306361
conf = append(conf, "key="+key)
307362
}
308-
for k, timeout := range map[string]int64{
363+
for k, timeout := range map[string]int{
309364
"rados_mon_op_timeout": radosMonOpTimeout,
310365
"rados_osd_op_timeout": radosOsdOpTimeout,
311366
"client_mount_timeout": clientMountTimeout,

0 commit comments

Comments
 (0)