diff --git a/image/manifest.go b/image/manifest.go
index 641e849..c438ede 100644
--- a/image/manifest.go
+++ b/image/manifest.go
@@ -88,7 +88,24 @@ func (m *manifest) validate(w walker) error {
 	return nil
 }
 
-func (m *manifest) unpack(w walker, dest string) error {
+func (m *manifest) unpack(w walker, dest string) (retErr error) {
+	// error out if the dest directory is not empty
+	s, err := ioutil.ReadDir(dest)
+	if err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "unable to open file") // err contains dest
+	}
+	if len(s) > 0 {
+		return fmt.Errorf("%s is not empty", dest)
+	}
+	defer func() {
+		// if we encounter error during unpacking
+		// clean up the partially-unpacked destination
+		if retErr != nil {
+			if err := os.RemoveAll(dest); err != nil {
+				fmt.Printf("Error: failed to remove partially-unpacked destination %v", err)
+			}
+		}
+	}()
 	for _, d := range m.Layers {
 		if d.MediaType != string(schema.MediaTypeImageLayer) {
 			continue
diff --git a/image/manifest_test.go b/image/manifest_test.go
index 350ef34..f50b2ed 100644
--- a/image/manifest_test.go
+++ b/image/manifest_test.go
@@ -122,3 +122,67 @@ func TestUnpackLayer(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestUnpackLayerRemovePartialyUnpackedFile(t *testing.T) {
+	// generate a tar file has duplicate entry which will failed on unpacking
+	tmp1, err := ioutil.TempDir("", "test-layer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp1)
+	err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar")
+	f, err := os.Create(tarfile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gw := gzip.NewWriter(f)
+	tw := tar.NewWriter(gw)
+
+	tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600})
+	io.Copy(tw, bytes.NewReader([]byte("test")))
+	tw.WriteHeader(&tar.Header{Name: "test", Size: 5, Mode: 0600})
+	io.Copy(tw, bytes.NewReader([]byte("test1")))
+	tw.Close()
+	gw.Close()
+	f.Close()
+
+	// generate sha256 hash
+	h := sha256.New()
+	file, err := os.Open(tarfile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	_, err = io.Copy(h, file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil))))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testManifest := manifest{
+		Layers: []descriptor{descriptor{
+			MediaType: "application/vnd.oci.image.layer.tar+gzip",
+			Digest:    fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))),
+		}},
+	}
+	err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs"))
+	if err != nil && !strings.Contains(err.Error(), "duplicate entry for") {
+		t.Fatal(err)
+	}
+
+	_, err = os.Stat(filepath.Join(tmp1, "rootfs"))
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatal(err)
+	}
+	if err == nil {
+		t.Fatal("Execpt partialy unpacked file has been removed")
+	}
+}