Skip to content

Commit 64adf37

Browse files
jayunit100jayunit100
committed
merge2
1 parent b1d40f6 commit 64adf37

File tree

5 files changed

+307
-47
lines changed

5 files changed

+307
-47
lines changed

‎README‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ INSTALLATION
4444

4545
# ls target/
4646
classes glusterfs-0.20.2-0.1.jar maven-archiver surefire-reports test-classes
47-
^^^^^^^^^^^^^^^^^^
4847

4948
Copy the plugin to lib/ directory in your $HADOOP_HOME dir.
5049

‎src/main/java/org/apache/hadoop/fs/glusterfs/FileInfoUtil.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package org.apache.hadoop.fs.glusterfs;
2+
3+
import java.io.DataInputStream;
4+
import java.io.IOException;
5+
import java.util.Map;
6+
import java.util.TreeMap;
7+
8+
/**
9+
* Because java 6 doesn't support cross platform file info, we use Runtime exec
10+
* to get details.
11+
*
12+
* This class will be extended in the future to also return permissions.
13+
*/
14+
public class FileInfoUtil {
15+
16+
/**
17+
* Returns the array of values from ls -aFl :
18+
*
19+
* Currently only handles "owner" , since that is what is most needed by
20+
* the GFAPI that is meant to use this class.
21+
*
22+
* m = getLSinfo("myfile.txt").get("owner")
23+
* System.out.println(m) ; <-- jayunit100
24+
*/
25+
public static Map<String, String> getLSinfo(String filename) throws IOException {
26+
String ret="";
27+
try {
28+
String ls_str;
29+
Process ls_proc = Runtime.getRuntime().exec("/bin/ls -aFl " + filename);
30+
DataInputStream ls_in = new DataInputStream(
31+
ls_proc.getInputStream());
32+
try {
33+
//should only be one line if input is a file
34+
while ((ls_str = ls_in.readLine()) != null) {
35+
if(ls_str.length()<10)
36+
;
37+
else{
38+
ret+=(ls_str);
39+
}
40+
}
41+
}
42+
catch (IOException e) {
43+
throw new RuntimeException(e);
44+
}
45+
}
46+
catch (IOException e1) {
47+
throw new RuntimeException(e1);
48+
}
49+
//initialization to "" was only to make for clean appending.
50+
if(ret.equals(""))
51+
ret=null;
52+
53+
//Convenience: transform to map
54+
Map<String, String> mm = lsToMap(ret);
55+
return mm;
56+
}
57+
58+
/**
59+
* Converts the output of ls -aFl to a map :
60+
*
61+
* EXPECTED INPUT STRING:
62+
* -rwxr-xr-x. 1 root root 786 Mar 7 11:42 buildbranch_and_copyjartovms.sh*
63+
*
64+
* EXPECTED OUTPUT:
65+
* key="owner" -> the owner of the file (root)
66+
* key="permissions" -> the hadoop permissions on the file.
67+
*/
68+
private static Map<String, String> lsToMap(String ret) {
69+
String[] values = ret.split("\\s+");
70+
71+
Map<String,String> mm = new TreeMap();
72+
mm.put("owner", values[2]);
73+
74+
return mm;
75+
}
76+
77+
78+
}

‎src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java‎

Lines changed: 139 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,13 @@
2121
* Implements the Hadoop FileSystem Interface to allow applications to store
2222
* files on GlusterFS and run Map/Reduce jobs on the data.
2323
*/
24-
2524
package org.apache.hadoop.fs.glusterfs;
2625

2726
import java.io.*;
2827
import java.net.*;
2928

29+
import java.util.StringTokenizer;
30+
import java.util.TreeMap;
3031
import java.util.regex.*;
3132

3233
import org.apache.hadoop.conf.Configuration;
@@ -39,8 +40,8 @@
3940
import org.apache.hadoop.fs.BlockLocation;
4041
import org.apache.hadoop.fs.permission.FsPermission;
4142
import org.apache.hadoop.util.Progressable;
42-
43-
import java.util.TreeMap;
43+
import org.apache.hadoop.util.Shell;
44+
import org.apache.hadoop.util.StringUtils;
4445

4546
/*
4647
* This package provides interface for hadoop jobs (incl. Map/Reduce)
@@ -76,23 +77,22 @@ public boolean FUSEMount(String volname, String server, String mount)
7677
boolean ret = true;
7778
int retVal = 0;
7879
Process p = null;
80+
String s = null;
7981
String mountCmd = null;
8082

81-
mountCmd = "mount -t glusterfs " + server + ":" + "/" + volname + " "+ mount;
82-
System.out.println("Running: " + mountCmd);
83+
mountCmd = "mount -t glusterfs " + server + ":" + "/" + volname + " "
84+
+ mount;
85+
System.out.println(mountCmd);
8386
try {
8487
p = Runtime.getRuntime().exec(mountCmd);
85-
8688
retVal = p.waitFor();
8789
if (retVal != 0)
8890
ret = false;
8991
}
9092
catch (IOException e) {
91-
e.printStackTrace();
92-
System.out.println("Error calling mount, Continuing.., hopefully its already been mounted.");
93-
//throw new RuntimeException("Problem mounting FUSE mount on: "+ mount);
93+
System.out.println("Problem mounting FUSE mount on: " + mount);
94+
throw new RuntimeException(e);
9495
}
95-
9696
return ret;
9797
}
9898

@@ -113,14 +113,19 @@ public void initialize(URI uri, Configuration conf) throws IOException {
113113
remoteGFSServer = conf.get("fs.glusterfs.server", null);
114114
needQuickRead = conf.get("quick.slave.io", null);
115115

116+
/*
117+
* bail out if we do not have enough information to do a FUSE mount
118+
*/
116119
if ((volName.length() == 0) || (remoteGFSServer.length() == 0)
117120
|| (glusterMount.length() == 0))
118-
throw new RuntimeException("Not enough info to mount FUSE: volname="+volName + " glustermount=" + glusterMount);
121+
throw new RuntimeException(
122+
"Not enough info for FUSE Mount : volname=" + volName
123+
+ ",server=" + remoteGFSServer
124+
+ ",glustermount=" + glusterMount);
119125

120-
121126
ret = FUSEMount(volName, remoteGFSServer, glusterMount);
122127
if (!ret) {
123-
throw new RuntimeException("Initialize: Failed to mount GlusterFS ");
128+
throw new RuntimeException("Failed to init Gluster FS");
124129
}
125130

126131
if((needQuickRead.length() != 0)
@@ -142,8 +147,7 @@ public void initialize(URI uri, Configuration conf) throws IOException {
142147
setConf(conf);
143148
}
144149
catch (Exception e) {
145-
e.printStackTrace();
146-
throw new RuntimeException("Unable to initialize GlusterFS " + e.getMessage());
150+
throw new RuntimeException(e);
147151
}
148152
}
149153

@@ -261,31 +265,137 @@ public FileStatus getFileStatusFromFileString(String path)
261265
return getFileStatus(nPath);
262266
}
263267

268+
public static class FUSEFileStatus extends FileStatus {
269+
File theFile;
270+
271+
public FUSEFileStatus(File f) {
272+
super();
273+
theFile = f;
274+
}
275+
276+
public FUSEFileStatus(File f, boolean isdir, int block_replication,
277+
long blocksize, Path path) {
278+
// if its a dir, 0 length
279+
super(isdir ? 0 : f.length(), isdir, block_replication, blocksize,
280+
f.lastModified(), path);
281+
theFile = f;
282+
}
283+
284+
/**
285+
* Wrapper to "ls -aFL" - this should fix BZ908898
286+
*/
287+
@Override
288+
public String getOwner() {
289+
try {
290+
return FileInfoUtil.getLSinfo(theFile.getAbsolutePath()).
291+
get("owner");
292+
}
293+
catch (Exception e) {
294+
throw new RuntimeException(e);
295+
}
296+
}
297+
298+
public FsPermission getPermission() {
299+
//should be amortized, see method.
300+
loadPermissionInfo();
301+
return super.getPermission();
302+
}
303+
304+
boolean permsLoaded=false;
305+
306+
/// loads permissions, owner, and group from `ls -ld`
307+
private void loadPermissionInfo() {
308+
if (permsLoaded) {
309+
return;
310+
}
311+
IOException e = null;
312+
try {
313+
String output;
314+
StringTokenizer t = new StringTokenizer(
315+
output=execCommand(theFile,
316+
Shell.getGET_PERMISSION_COMMAND()));
317+
318+
//System.out.println("Output of PERMISSION command = " + output + " for " + this.getPath());
319+
//expected format
320+
//-rw------- 1 username groupname ...
321+
String permission = t.nextToken();
322+
if (permission.length() > 10) { //files with ACLs might have a '+'
323+
permission = permission.substring(0, 10);
324+
}
325+
setPermission(FsPermission.valueOf(permission));
326+
t.nextToken();
327+
setOwner(t.nextToken());
328+
setGroup(t.nextToken());
329+
} catch (Shell.ExitCodeException ioe) {
330+
if (ioe.getExitCode() != 1) {
331+
e = ioe;
332+
} else {
333+
setPermission(null);
334+
setOwner(null);
335+
setGroup(null);
336+
}
337+
permsLoaded=true;
338+
}
339+
catch (IOException ioe) {
340+
e = ioe;
341+
}
342+
finally {
343+
if (e != null) {
344+
throw new RuntimeException("Error while running command to get " +
345+
"file permissions : " +
346+
StringUtils.stringifyException(e));
347+
}
348+
}
349+
}
350+
351+
}
352+
353+
public static String execCommand(File f, String... cmd) throws IOException {
354+
String[] args = new String[cmd.length + 1];
355+
System.arraycopy(cmd, 0, args, 0, cmd.length);
356+
args[cmd.length] = f.getCanonicalPath();
357+
String output = Shell.execCommand(args);
358+
return output;
359+
}
360+
361+
/**
362+
* We ultimately use chmod to set permissions, same as in
363+
* https://svn.apache.org/repos/asf/hadoop/common/branches/HADOOP-3628/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
364+
*/
365+
@Override
366+
public void setPermission(Path p, FsPermission permission){
367+
try{
368+
Path absolute = makeAbsolute(p);
369+
final File f = new File(absolute.toUri().getPath());
370+
371+
execCommand(f, Shell.SET_PERMISSION_COMMAND,
372+
String.format("%05o", permission.toShort()));
373+
}
374+
catch(Exception e){
375+
throw new RuntimeException(e);
376+
}
377+
}
378+
379+
264380
public FileStatus getFileStatus(Path path) throws IOException {
265381
Path absolute = makeAbsolute(path);
266-
File f = new File(absolute.toUri().getPath());
382+
final File f = new File(absolute.toUri().getPath());
267383

268384
if (!f.exists())
269385
throw new FileNotFoundException("File " + f.getPath()
270386
+ " does not exist.");
271-
272387
FileStatus fs;
388+
273389
// simple version - should work . we'll see.
390+
// TODO COMPARE these w/ original signatures - do we retain the correct
391+
// default args?
274392
if (f.isDirectory())
275-
fs = new FileStatus(0, true, 1, 0, f.lastModified(),
276-
path.makeQualified(this)) {
277-
public String getOwner() {
278-
return "root";
279-
}
280-
};
393+
fs = new FUSEFileStatus(f, true, 1, 0, path.makeQualified(this));
281394
else
282-
fs = new FileStatus(f.length(), false, 0, getDefaultBlockSize(),
283-
f.lastModified(), path.makeQualified(this)) {
284-
public String getOwner() {
285-
return "root";
286-
}
287-
};
395+
fs = new FUSEFileStatus(f, false, 0, getDefaultBlockSize(),
396+
path.makeQualified(this));
288397
return fs;
398+
289399
}
290400

291401
/*

‎src/test/java/org/gluster/test/TestFileInfo.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package org.gluster.test;
2+
3+
import java.io.File;
4+
5+
import junit.framework.Assert;
6+
7+
import org.apache.hadoop.fs.glusterfs.FileInfoUtil;
8+
import org.apache.tools.ant.types.FlexInteger;
9+
import org.junit.Test;
10+
11+
public class TestFileInfo {
12+
13+
/**
14+
* This is a unit test of the FileInfoUtils ability
15+
* to read and parse permissions from POSIX and run runtime.exec correctly.
16+
* It is not a test of gluster. For a test of the GlusterFileSystem ownership, see
17+
* {@link TestGluster} .testOwner.
18+
*/
19+
@Test
20+
public void testPosix() throws Exception {
21+
String user=System.getProperties().getProperty("user.name");
22+
File f = File.createTempFile("tempjunit", ".tmp");
23+
String owner=FileInfoUtil.getLSinfo(f.getAbsolutePath()).get("owner");
24+
System.out.println("Confirming -- \nuser.name(" + user +")=owner("+owner+")");
25+
Assert.assertEquals(user,owner);
26+
}
27+
28+
29+
}
30+

0 commit comments

Comments
 (0)