Skip to content

Commit 2ecf673

Browse files
childsbchildsb
committed
Merge pull request #51 from childsb/master
Remove CRC from main glusterfs classes. added extra classes for CRC.
2 parents 179d4d9 + 9b9e3ce commit 2ecf673

File tree

11 files changed

+266
-44
lines changed

11 files changed

+266
-44
lines changed

‎conf/core-site.xml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,39 @@
55

66
<configuration>
77

8+
<!-- default hadoop 1.x glusterfs with no additional integrity checking on files -->
89
<property>
910
<name>fs.glusterfs.impl</name>
1011
<value>org.apache.hadoop.fs.glusterfs.GlusterFileSystem</value>
1112
</property>
1213

14+
<!-- this is an optional hadoop 1.x glusterfs which does extra integrity checks on files. -->
15+
<!--
16+
<property>
17+
<name>fs.glusterfs.impl</name>
18+
<value>org.apache.hadoop.fs.glusterfs.GlusterFileSystemCRC</value>
19+
</property>
20+
-->
21+
1322
<property>
1423
<name>fs.default.name</name>
1524
<value>glusterfs://ambari-3.abrv8.com:gv0<</value>
1625
</property>
1726

27+
28+
<!-- this is the default glusterfs hook with no additional integrity checking on files -->
1829
<property>
1930
<name>fs.AbstractFileSystem.glusterfs.impl</name>
2031
<value>org.apache.hadoop.fs.local.GlusterFs</value>
2132
</property>
2233

23-
34+
<!-- this is an optional glusterfs which does extra integrity checks on files. -->
35+
<!--
36+
<property>
37+
<name>fs.AbstractFileSystem.glusterfs.impl</name>
38+
<value>org.apache.hadoop.fs.local.GlusterFsCRC</value>
39+
</property>
40+
-->
2441

2542
<property>
2643
<name>fs.glusterfs.volname</name>

‎src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileStatus.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public String getGroup(){
8484
private void loadPermissionInfo(){
8585
IOException e=null;
8686
try{
87-
StringTokenizer t=new StringTokenizer(Util.execCommand(fs.pathToFile(getPath()), Shell.getGET_PERMISSION_COMMAND()));
87+
StringTokenizer t=new StringTokenizer(Util.execCommand(fs.pathToFile(getPath()), Util.getGET_PERMISSION_COMMAND()));
8888
// expected format
8989
// -rw------- 1 username groupname ...
9090
String permission=t.nextToken();

‎src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java‎

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,57 +19,98 @@
1919

2020
/**
2121
* Implements the Hadoop FileSystem Interface to allow applications to store
22-
* files on GlusterFS and run Map/Reduce jobs on the data.
22+
* files on GlusterFS and run Map/Reduce jobs on the data. This code does NOT perform a CRC
23+
* on files.
2324
*
2425
* gluster file systems are specified with the glusterfs:// prefix.
2526
*
2627
*
2728
*/
2829
package org.apache.hadoop.fs.glusterfs;
2930

31+
import java.io.File;
3032
import java.io.IOException;
33+
import java.net.URI;
3134

3235
import org.apache.hadoop.conf.Configuration;
3336
import org.apache.hadoop.fs.FileSystem;
3437
import org.apache.hadoop.fs.FileUtil;
35-
import org.apache.hadoop.fs.LocalFileSystem;
38+
import org.apache.hadoop.fs.FilterFileSystem;
3639
import org.apache.hadoop.fs.Path;
3740
import org.slf4j.Logger;
3841
import org.slf4j.LoggerFactory;
3942

43+
public class GlusterFileSystem extends FilterFileSystem{
44+
45+
protected static final Logger log=LoggerFactory.getLogger(GlusterFileSystem.class);
46+
FileSystem rfs;
47+
String swapScheme;
48+
49+
public FileSystem getRaw(){
50+
return rfs;
51+
}
52+
53+
public void initialize(URI name,Configuration conf) throws IOException{
54+
if(fs.getConf()==null){
55+
fs.initialize(name, conf);
56+
}
57+
String scheme=name.getScheme();
58+
if(!scheme.equals(fs.getUri().getScheme())){
59+
swapScheme=scheme;
60+
}
61+
}
4062

41-
public class GlusterFileSystem extends LocalFileSystem{
42-
43-
protected static final Logger log = LoggerFactory.getLogger(GlusterFileSystem.class);
44-
4563
public GlusterFileSystem(){
46-
super(new GlusterVolume());
64+
this(new GlusterVolume());
65+
log.info("Initializing GlusterFS, CRC disabled.");
66+
}
67+
68+
public GlusterFileSystem(FileSystem rawLocalFileSystem){
69+
super(rawLocalFileSystem);
70+
rfs=rawLocalFileSystem;
71+
}
72+
73+
/** Convert a path to a File. */
74+
public File pathToFile(Path path){
75+
return ((GlusterVolume) fs).pathToFile(path);
76+
}
77+
78+
/**
79+
* Get file status.
80+
*/
81+
public boolean exists(Path f) throws IOException{
82+
File path=pathToFile(f);
83+
if(path.exists()){
84+
return true;
85+
}else{
86+
return false;
87+
}
4788
}
48-
89+
4990
public void setConf(Configuration conf){
50-
log.info("Initializing GlusterFS");
91+
log.info("Configuring GlusterFS");
5192
super.setConf(conf);
5293
}
5394

5495
/*
55-
* if GlusterFileSystem is the default filesystem, real local URLs come back without a file:/ scheme name (BUG!). the glusterfs
56-
* file system is assumed. force a schema.
96+
* if GlusterFileSystem is the default filesystem, real local URLs come back
97+
* without a file:/ scheme name (BUG!). the glusterfs file system is
98+
* assumed. force a schema.
5799
*/
58-
@Override
59-
public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
60-
FileSystem srcFs = new Path("file:/" + src.toString()).getFileSystem(getConf());
61-
FileSystem dstFs = dst.getFileSystem(getConf());
100+
101+
public void copyFromLocalFile(boolean delSrc,Path src,Path dst) throws IOException{
102+
FileSystem srcFs=new Path("file:/"+src.toString()).getFileSystem(getConf());
103+
FileSystem dstFs=dst.getFileSystem(getConf());
62104
FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
63105
}
64106

65-
@Override
66-
public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
67-
FileSystem srcFs = src.getFileSystem(getConf());
68-
FileSystem dstFs = new Path("file:/" + dst.toString()).getFileSystem(getConf());
69-
FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
107+
public void copyToLocalFile(boolean delSrc,Path src,Path dst) throws IOException{
108+
FileSystem srcFs=src.getFileSystem(getConf());
109+
FileSystem dstFs=new Path("file:/"+dst.toString()).getFileSystem(getConf());
110+
FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
70111
}
71-
112+
72113
public String toString(){
73-
return "Gluster File System";
114+
return "Gluster File System, no CRC.";
74115
}
75116
}

‎src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/**
2+
* Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3+
* This file is part of GlusterFS.
4+
*
5+
* Licensed under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14+
* implied. See the License for the specific language governing
15+
* permissions and limitations under the License.
16+
*
17+
*/
18+
19+
/**
20+
* Implements the Hadoop FileSystem Interface to allow applications to store
21+
* files on GlusterFS and run Map/Reduce jobs on the data. CRC are performed on files.
22+
*
23+
* gluster file systems are specified with the glusterfs:// prefix.
24+
*
25+
*
26+
*/
27+
package org.apache.hadoop.fs.glusterfs;
28+
29+
import java.io.IOException;
30+
31+
import org.apache.hadoop.conf.Configuration;
32+
import org.apache.hadoop.fs.FileSystem;
33+
import org.apache.hadoop.fs.FileUtil;
34+
import org.apache.hadoop.fs.LocalFileSystem;
35+
import org.apache.hadoop.fs.Path;
36+
import org.slf4j.Logger;
37+
import org.slf4j.LoggerFactory;
38+
39+
40+
public class GlusterFileSystemCRC extends LocalFileSystem{
41+
42+
protected static final Logger log = LoggerFactory.getLogger(GlusterFileSystemCRC.class);
43+
44+
public GlusterFileSystemCRC(){
45+
super(new GlusterVolume());
46+
}
47+
48+
public void setConf(Configuration conf){
49+
log.info("Initializing GlusterFS with CRC.");
50+
super.setConf(conf);
51+
}
52+
53+
/*
54+
* if GlusterFileSystem is the default filesystem, real local URLs come back without a file:/ scheme name (BUG!). the glusterfs
55+
* file system is assumed. force a schema.
56+
*/
57+
@Override
58+
public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
59+
FileSystem srcFs = new Path("file:/" + src.toString()).getFileSystem(getConf());
60+
FileSystem dstFs = dst.getFileSystem(getConf());
61+
FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
62+
}
63+
64+
@Override
65+
public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException {
66+
FileSystem srcFs = src.getFileSystem(getConf());
67+
FileSystem dstFs = new Path("file:/" + dst.toString()).getFileSystem(getConf());
68+
FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf());
69+
}
70+
71+
public String toString(){
72+
return "Gluster File System - CRC Enabled";
73+
}
74+
}

‎src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737

3838
public class GlusterVolume extends RawLocalFileSystem{
3939

40-
static final Logger log = LoggerFactory.getLogger(GlusterFileSystem.class);
40+
static final Logger log = LoggerFactory.getLogger(GlusterFileSystemCRC.class);
4141
public static final URI NAME = URI.create("glusterfs:///");
4242

4343
protected String root=null;
@@ -53,7 +53,7 @@ public GlusterVolume(Configuration conf){
5353
public URI getUri() { return NAME; }
5454

5555
public void setConf(Configuration conf){
56-
log.info("initializing gluster volume..");
56+
log.info("Initializing gluster volume..");
5757
super.setConf(conf);
5858
String getfattrcmd = null;
5959
if(conf!=null){

‎src/main/java/org/apache/hadoop/fs/glusterfs/Util.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,23 @@
3131

3232
public class Util{
3333

34-
public static String execCommand(File f, String... cmd) throws IOException {
35-
String[] args = new String[cmd.length + 1];
36-
System.arraycopy(cmd, 0, args, 0, cmd.length);
37-
args[cmd.length] = FileUtil.makeShellPath(f, true);
38-
String output = Shell.execCommand(args);
39-
return output;
40-
}
34+
public static String execCommand(File f,String...cmd) throws IOException{
35+
String[] args=new String[cmd.length+1];
36+
System.arraycopy(cmd, 0, args, 0, cmd.length);
37+
args[cmd.length]=FileUtil.makeShellPath(f, true);
38+
String output=Shell.execCommand(args);
39+
return output;
40+
}
4141

42+
/* copied from unstalbe hadoop API org.apache.hadoop.Shell */
43+
public static String[] getGET_PERMISSION_COMMAND(){
44+
// force /bin/ls, except on windows.
45+
return new String[]{(WINDOWS ? "ls" : "/bin/ls"),"-ld"};
46+
}
47+
48+
/* copied from unstalbe hadoop API org.apache.hadoop.Shell */
49+
50+
public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
51+
=System.getProperty("os.name").startsWith("Windows");
52+
// / loads permissions, owner, and group from `ls -ld`
4253
}

‎src/main/java/org/apache/hadoop/fs/local/GlusterFs.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,42 @@
1+
/**
2+
*
3+
* Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
4+
* This file is part of GlusterFS.
5+
*
6+
* Licensed under the Apache License, Version 2.0
7+
* (the "License"); you may not use this file except in compliance with
8+
* the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15+
* implied. See the License for the specific language governing
16+
* permissions and limitations under the License.
17+
*
18+
*/
19+
20+
/**
21+
* Implements the Hadoop FileSystem 2.x Interface to allow applications to store
22+
* files on GlusterFS and run Map/Reduce jobs on the data. This code does not perform a CRC
23+
* on the files.
24+
*
25+
* gluster file systems are specified with the glusterfs:// prefix.
26+
*
27+
*
28+
*/
29+
130
package org.apache.hadoop.fs.local;
231

332
import java.io.IOException;
433
import java.net.URI;
534
import java.net.URISyntaxException;
635

736
import org.apache.hadoop.conf.Configuration;
8-
import org.apache.hadoop.fs.ChecksumFs;
37+
import org.apache.hadoop.fs.FilterFs;
938

10-
public class GlusterFs extends ChecksumFs{
39+
public class GlusterFs extends FilterFs{
1140

1241
GlusterFs(Configuration conf) throws IOException, URISyntaxException{
1342
super(new GlusterVol(conf));
@@ -17,4 +46,5 @@ public class GlusterFs extends ChecksumFs{
1746
this(conf);
1847
}
1948

49+
2050
}

‎src/main/java/org/apache/hadoop/fs/local/GlusterFsCRC.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/**
2+
*
3+
* Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
4+
* This file is part of GlusterFS.
5+
*
6+
* Licensed under the Apache License, Version 2.0
7+
* (the "License"); you may not use this file except in compliance with
8+
* the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
15+
* implied. See the License for the specific language governing
16+
* permissions and limitations under the License.
17+
*
18+
*/
19+
20+
/**
21+
* Implements the Hadoop FileSystem 2.x Interface to allow applications to store
22+
* files on GlusterFS and run Map/Reduce jobs on the data. This code does perform a CRC
23+
* on the files.
24+
*
25+
* gluster file systems are specified with the glusterfs:// prefix.
26+
*
27+
*
28+
*/
29+
30+
package org.apache.hadoop.fs.local;
31+
32+
import java.io.IOException;
33+
import java.net.URI;
34+
import java.net.URISyntaxException;
35+
36+
import org.apache.hadoop.conf.Configuration;
37+
import org.apache.hadoop.fs.ChecksumFs;
38+
39+
public class GlusterFsCRC extends ChecksumFs{
40+
41+
GlusterFsCRC(Configuration conf) throws IOException, URISyntaxException{
42+
super(new GlusterVol(conf));
43+
}
44+
45+
GlusterFsCRC(final URI theUri, final Configuration conf) throws IOException, URISyntaxException{
46+
this(conf);
47+
}
48+
49+
}

0 commit comments

Comments
 (0)