add new feature that directly importing directories from skopeo.

Author: JasonYangShadow

build/linux/x86_64/Linux-x86_64-lpmx

@@ -1751,6 +1751,157 @@ func SingularityLoad(file string, name string, tag string) *Error {
 	}
 }
 
+//name -> name with tag info
+func SkopeoLoad(name, dir string) *Error {
+	currdir, err := GetConfigDir()
+	if err != nil {
+		return err
+	}
+	rootdir := fmt.Sprintf("%s/.lpmxdata", currdir)
+	sysdir := fmt.Sprintf("%s/.lpmxsys", currdir)
+	tempdir := fmt.Sprintf("%s/.temp", currdir)
+	//we delete temp dir if it exists at the end of the function
+	defer func() {
+		if FolderExist(tempdir) {
+			os.RemoveAll(tempdir)
+		}
+	}()
+
+	//check if folder exists
+	adir, derr := filepath.Abs(dir)
+	if derr!= nil {
+		cerr := ErrNew(derr, fmt.Sprintf("could not parse the absolute path: %s", adir))
+		return cerr
+	}
+	if !FolderExist(adir) {
+		cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", adir))
+		return cerr
+	}
+
+	//configure Docker related info locally
+	var doc Image
+	err = unmarshalObj(rootdir, &doc)
+	if err != nil && err.Err != ErrNExist {
+		return err
+	}
+
+	if err != nil && err.Err == ErrNExist {
+		ret, err := MakeDir(rootdir)
+		doc.RootDir = rootdir
+		doc.Images = make(map[string]interface{})
+		if !ret {
+			return err
+		}
+	}
+
+	image_dir := fmt.Sprintf("%s/.image", rootdir)
+	ret, layer_order, lerr := LoadSkopeoTar(adir, image_dir)
+	if lerr != nil {
+		return lerr
+	}
+	if _, ok := doc.Images[name]; ok {
+		cerr := ErrNew(ErrExist, fmt.Sprintf("%s already exists", name))
+		return cerr
+	} else {
+		tdata := strings.Split(name, ":")
+		tname := tdata[0]
+		ttag := tdata[1]
+		mdata := make(map[string]interface{})
+		mdata["rootdir"] = fmt.Sprintf("%s/%s/%s", doc.RootDir, tname, ttag)
+		mdata["config"] = fmt.Sprintf("%s/setting.yml", mdata["rootdir"].(string))
+		mdata["image"] = fmt.Sprintf("%s/.image", rootdir)
+		mdata["layer"] = ret
+		mdata["layer_order"] = strings.Join(layer_order, ":")
+		mdata["imagetype"] = "Docker"
+
+		//add docker info file(.info)
+		if !FolderExist(mdata["rootdir"].(string)) {
+			merr := os.MkdirAll(mdata["rootdir"].(string), os.FileMode(FOLDER_MODE))
+			if merr != nil {
+				cerr := ErrNew(merr, fmt.Sprintf("could not mkdir %s", mdata["rootdir"].(string)))
+				return cerr
+			}
+		}
+		var docinfo ImageInfo
+		docinfo.Name = name
+		docinfo.ImageType = "Docker"
+		// layer_order is absolute path
+		//docinfo layers map should remove absolute path of host
+		layersmap := make(map[string]int64)
+		for k, v := range ret {
+			layersmap[path.Base(k)] = v
+		}
+		docinfo.LayersMap = layersmap
+
+		var layer_sha []string
+		for _, layer := range layer_order {
+			layer_sha = append(layer_sha, path.Base(layer))
+		}
+		docinfo.Layers = strings.Join(layer_sha, ":")
+
+		LOGGER.WithFields(logrus.Fields{
+			"docinfo": docinfo,
+		}).Debug("Skopeo debug, docinfo debug")
+
+		dinfodata, _ := StructMarshal(docinfo)
+		err = WriteToFile(dinfodata, fmt.Sprintf("%s/.info", mdata["rootdir"].(string)))
+		if err != nil {
+			return err
+		}
+		//end
+
+		workspace := fmt.Sprintf("%s/workspace", mdata["rootdir"])
+		if !FolderExist(workspace) {
+			MakeDir(workspace)
+		}
+		mdata["workspace"] = workspace
+
+		//extract layers
+		base := fmt.Sprintf("%s/.base", rootdir)
+		if !FolderExist(base) {
+			MakeDir(base)
+		}
+		mdata["base"] = base
+
+		for _, k := range layer_order {
+			k = path.Base(k)
+			tar_path := fmt.Sprintf("%s/%s", image_dir, k)
+			layerfolder := fmt.Sprintf("%s/%s", mdata["base"], k)
+			if !FolderExist(layerfolder) {
+				MakeDir(layerfolder)
+			}
+
+			err := Untar(tar_path, layerfolder)
+			if err != nil {
+				return err
+			}
+		}
+
+		//download setting from github
+		rdir, _ := mdata["rootdir"].(string)
+
+		yaml := fmt.Sprintf("%s/distro.management.yml", sysdir)
+		err = DownloadFilefromGithubPlus(tname, ttag, "setting.yml", SETTING_URL, rdir, yaml)
+		if err != nil {
+			LOGGER.WithFields(logrus.Fields{
+				"err":    err,
+				"toPath": rdir,
+			}).Error("Download setting from github failure and could not rollback to default one")
+			return err
+		}
+
+		//add map to this image
+		doc.Images[name] = mdata
+
+		ddata, _ := StructMarshal(doc)
+		err = WriteToFile(ddata, fmt.Sprintf("%s/.info", doc.RootDir))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func DockerLoad(file string) *Error {
 	currdir, err := GetConfigDir()
 	if err != nil {
@@ -1767,8 +1918,13 @@ func DockerLoad(file string) *Error {
 	}()
 
 	//check if file exists
-	if !FileExist(file) {
-		cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", file))
+	afile, ferr := filepath.Abs(file)
+	if ferr != nil {
+		cerr := ErrNew(ferr, fmt.Sprintf("could not parse to the absolute path: %s", file))
+		return cerr
+	}
+	if !FileExist(afile) {
+		cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", afile))
 		return cerr
 	}
 
@@ -1795,7 +1951,7 @@ func DockerLoad(file string) *Error {
 	}
 
 	//untar tar ball
-	uerr := Untar(file, tmpdir)
+	uerr := Untar(afile, tmpdir)
 	if uerr != nil {
 		return uerr
 	}

container/container.go

@@ -38,6 +38,20 @@ type DockerSaveInfo struct {
 	Layers   []string //layers included inside this image, from lower to higher layers
 }
 
+//Skopeo manifest item structure
+type SkopeoManifestItem struct {
+	MediaType string `json:"mediaType"`
+	Size uint `json:"size"`
+	Digest string `json:"digest"`
+}
+//Skopeo manifest structure
+type SkopeoManifest struct {
+	SchemaVersion int `json:"schemaVersion"`
+	MediaType string `json:"mediaType"`
+	Config SkopeoManifestItem `json:"config"`
+	Layers []SkopeoManifestItem `json:"layers"`
+}
+
 func ListRepositories(username string, pass string) ([]string, *Error) {
 	log.SetOutput(ioutil.Discard)
 	hub, err := registry.New(DOCKER_URL, username, pass)
@@ -342,6 +356,47 @@ func DeleteManifest(username string, pass string, name string, tag string) *Erro
 	return nil
 }
 
+func LoadSkopeoTar(dir, imagedir string) (map[string]int64, []string, *Error) {
+	if !FolderExist(dir) {
+		cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does not exist", dir))
+		return nil, nil, cerr
+	}
+
+	var info SkopeoManifest
+	manifest_file := fmt.Sprintf("%s/manifest.json", dir)
+	b, berr := ioutil.ReadFile(manifest_file)
+	if berr != nil {
+		cerr := ErrNew(berr, fmt.Sprintf("could not read file: %s", manifest_file))
+		return nil, nil, cerr
+	}
+
+	jerr := json.Unmarshal(b, &info)
+	if jerr != nil {
+		cerr := ErrNew(jerr, "could not unmarshal json bytes to objects")
+		return nil, nil, cerr
+	}
+
+	layer_data := make(map[string]int64)
+	var layers []string
+	for _, item := range info.Layers {
+		shavalue := strings.Split(item.Digest, ":")[1]
+		layer_path := fmt.Sprintf("%s/%s", dir, shavalue)
+		target_path := fmt.Sprintf("%s/%s.tar.gz", imagedir, shavalue)
+		_, rerr := CopyFile(layer_path, target_path)
+		if rerr != nil {
+			return nil, nil, rerr
+		}
+		file_length, ferr := GetFileLength(target_path)
+		if ferr != nil {
+			return nil, nil, ferr
+		}
+		layer_data[target_path] = file_length
+		layers = append(layers, target_path)
+	}
+
+	return layer_data, layers, nil
+}
+
 //dir is temp dir used for tarball extraction, image dir is used for the storage of image data
 func LoadDockerTar(dir, imagedir string) (string, map[string]int64, []string, *Error) {
 	if !FolderExist(dir) {

docker/docker.go

@@ -20,7 +20,7 @@ var (
 )
 
 const (
-	VERSION = "alpha-1.7.2"
+	VERSION = "alpha-1.8"
 )
 
 func checkCompleteness() *Error {
@@ -672,12 +672,39 @@ func main() {
 		},
 	}
 
+	var SkopeoNameTag string
+	var skopeoLoadCmd = &cobra.Command{
+		Use:   "skopeoload",
+		Short: "load the skopeo directory",
+		Long:  "skopeoload sub-command is one advanced command of lpmx, which is used for importing 'skopeo copy' generated directory to system",
+		Args:  cobra.ExactArgs(1),
+		PreRun: func(cmd *cobra.Command, args []string) {
+			err := checkCompleteness()
+			if err != nil {
+				LOGGER.Fatal(err.Error())
+				return
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			err := SkopeoLoad(SkopeoNameTag, args[0])
+			if err != nil {
+				LOGGER.Fatal(err.Error())
+				return
+			} else {
+				LOGGER.Info("DONE")
+				return
+			}
+		},
+	}
+	skopeoLoadCmd.Flags().StringVarP(&SkopeoNameTag, "nametag", "n", "", "required")
+	skopeoLoadCmd.MarkFlagRequired("nametag")
+
 	var dockerCmd = &cobra.Command{
 		Use:   "docker",
 		Short: "docker command",
 		Long:  "docker command is the advanced command of lpmx, which is used for executing docker related commands",
 	}
-	dockerCmd.AddCommand(dockerCreateCmd, dockerSearchCmd, dockerListCmd, dockerDeleteCmd, dockerDownloadCmd, dockerResetCmd, dockerPackageCmd, dockerAddCmd, dockerCommitCmd, dockerLoadCmd, dockerRunCmd, dockerMergeCmd)
+	dockerCmd.AddCommand(dockerCreateCmd, dockerSearchCmd, dockerListCmd, dockerDeleteCmd, dockerDownloadCmd, dockerResetCmd, dockerPackageCmd, dockerAddCmd, dockerCommitCmd, dockerLoadCmd, dockerRunCmd, dockerMergeCmd, skopeoLoadCmd)
 
 	var SingularityLoadName string
 	var SingularityLoadTag string

main.go

@@ -288,18 +288,8 @@ func RemoveFile(path string) (bool, *Error) {
 
 func Rename(old_path string, new_path string) *Error {
 	if t, terr := FileType(old_path); terr == nil {
-		if t == TYPE_DIR {
-			parent_dir := filepath.Dir(new_path)
-			if !FolderExist(parent_dir) {
-				err := os.MkdirAll(parent_dir, 0777)
-				if err != nil {
-					cerr := ErrNew(err, fmt.Sprintf("could not mkdir %s", parent_dir))
-					return cerr
-				}
-			}
-		}
-		if t == TYPE_REGULAR {
-			parent_dir := filepath.Dir(new_path)
+		parent_dir := filepath.Dir(new_path)
+		if t == TYPE_DIR || t == TYPE_REGULAR {
 			if !FolderExist(parent_dir) {
 				err := os.MkdirAll(parent_dir, 0777)
 				if err != nil {
@@ -411,6 +401,14 @@ func CopyFile(src string, dst string) (bool, *Error) {
 		return false, cerr
 	}
 	defer in.Close()
+	parent_dir := filepath.Dir(dst)
+	if !FolderExist(parent_dir) {
+		err := os.MkdirAll(parent_dir, 0777)
+		if err != nil {
+			cerr := ErrNew(err, fmt.Sprintf("could not mkdir %s", parent_dir))
+			return false, cerr
+		}
+	}
 	out, oerr := os.Create(dst)
 	if oerr != nil {
 		cerr := ErrNew(oerr, fmt.Sprintf("can't open file %s", dst))