Skip to content

Commit 2ee4953

Browse files
jamojamo
committed
Unzipper can remove files when updating exercise
1 parent 0bc79fa commit 2ee4953

File tree

9 files changed

+198
-41
lines changed

9 files changed

+198
-41
lines changed

‎Dockerfile

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,12 @@
1-
FROM java:8u91
1+
FROM maven:3-jdk-8
22

3-
4-
ENV MAVEN_VERSION 3.3.9
5-
6-
RUN mkdir -p /usr/share/maven \
7-
&& curl -fsSL http://apache.osuosl.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz \
8-
| tar -xzC /usr/share/maven --strip-components=1 \
9-
&& ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
10-
11-
ENV MAVEN_HOME /usr/share/maven
3+
RUN apt-get update && apt-get install -y --no-install-recommends build-essential valgrind check pkg-config python3.4 && rm -rf /var/lib/apt/lists/*
124

135
RUN useradd -g users user && mkdir -p /home/user && chown -R user:users /home/user
14-
RUN apt-get update && apt-get install -y ruby build-essential valgrind check pkg-config python3.4 && rm -rf /var/lib/apt/lists/*
15-
16-
176
#RUN chown -R user:users /app
18-
#WORKDIR app
7+
WORKDIR /app
198

9+
RUN chown -R user:users /app
2010
USER user
2111

2212
CMD ["/bin/bash"]

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/ConfigurableStudentFilePolicy.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,4 +99,9 @@ private void loadExtraStudentFileList() {
9999
extraStudentFiles = parser.parseExtraStudentFiles(configFile, rootPath);
100100
}
101101
}
102+
103+
@Override
104+
public boolean mayDelete(Path file, Path projectRoot) {
105+
return false;
106+
}
102107
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/EverythingIsStudentFileStudentFilePolicy.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,9 @@ public final class EverythingIsStudentFileStudentFilePolicy implements StudentFi
1111
public boolean isStudentFile(Path path, Path projectRootPath) {
1212
return true;
1313
}
14+
15+
@Override
16+
public boolean mayDelete(Path file, Path projectRoot) {
17+
return false;
18+
}
1419
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/NothingIsStudentFileStudentFilePolicy.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,9 @@ public final class NothingIsStudentFileStudentFilePolicy implements StudentFileP
1111
public boolean isStudentFile(Path path, Path projectRootPath) {
1212
return false;
1313
}
14+
15+
@Override
16+
public boolean mayDelete(Path file, Path projectRoot) {
17+
return !isStudentFile(file, projectRoot);
18+
}
1419
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/StudentFilePolicy.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,10 @@ public interface StudentFilePolicy {
1515
* Answers whether the file in <tt>path</tt> is an student file.
1616
*/
1717
boolean isStudentFile(Path path, Path projectRootPath);
18+
19+
/**
20+
* Answers whether the file can be deleted, e.g. when extracting latest own submission or
21+
* when replacing with model solution.
22+
*/
23+
boolean mayDelete(Path file, Path projectRoot);
1824
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/zip/StudentFileAwareUnzipper.java

Lines changed: 103 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,26 @@
22

33
import fi.helsinki.cs.tmc.langs.io.StudentFilePolicy;
44

5+
import com.google.common.collect.Sets;
56
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
67
import org.apache.commons.compress.archivers.zip.ZipFile;
78
import org.apache.commons.io.FileUtils;
89

10+
import org.apache.commons.io.IOUtils;
911
import org.slf4j.Logger;
1012
import org.slf4j.LoggerFactory;
1113

14+
import java.io.BufferedInputStream;
15+
import java.io.ByteArrayInputStream;
1216
import java.io.File;
17+
import java.io.FileInputStream;
1318
import java.io.FileNotFoundException;
1419
import java.io.IOException;
1520
import java.io.InputStream;
1621
import java.nio.file.Files;
1722
import java.nio.file.Path;
1823
import java.util.Enumeration;
24+
import java.util.Set;
1925
import java.util.zip.ZipEntry;
2026

2127
public final class StudentFileAwareUnzipper implements Unzipper {
@@ -24,7 +30,8 @@ public final class StudentFileAwareUnzipper implements Unzipper {
2430

2531
private StudentFilePolicy filePolicy;
2632

27-
public StudentFileAwareUnzipper() {}
33+
public StudentFileAwareUnzipper() {
34+
}
2835

2936
public StudentFileAwareUnzipper(StudentFilePolicy filePolicy) {
3037
this.filePolicy = filePolicy;
@@ -36,7 +43,9 @@ public void setStudentFilePolicy(StudentFilePolicy studentFilePolicy) {
3643
}
3744

3845
@Override
39-
public void unzip(Path zip, Path target) throws IOException {
46+
public UnzipResult unzip(Path zip, Path target) throws IOException {
47+
UnzipResult result = new UnzipResult(target);
48+
4049
log.info("Unzipping {} to {}", zip, target);
4150
if (!Files.exists(zip)) {
4251
log.error("Attempted to unzip nonexistent archive {}", zip);
@@ -48,54 +57,107 @@ public void unzip(Path zip, Path target) throws IOException {
4857
Files.createDirectories(target);
4958
}
5059

60+
Set<Path> pathsInZip = Sets.newHashSet();
5161
try (ZipFile zipFile = new ZipFile(zip.toFile())) {
5262

5363
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
5464

5565
String projectDirInZip = findProjectDirInZip(zipFile.getEntries());
56-
if (projectDirInZip == null) {
57-
throw new RuntimeException("No project in zip");
58-
}
66+
5967
log.debug("Project dir in zip: {}", projectDirInZip);
6068

6169
while (entries.hasMoreElements()) {
6270
ZipArchiveEntry entry = entries.nextElement();
6371

6472
if (entry.getName().startsWith(projectDirInZip)) {
65-
String restOfPath = entry.getName().substring(projectDirInZip.length());
66-
restOfPath = trimSlashes(restOfPath);
67-
68-
String destFileRelativePath =
69-
trimSlashes(restOfPath.replace("/", File.separator));
70-
Path entryTargetPath = target.resolve(destFileRelativePath);
71-
72-
log.debug(
73-
"Processing zipEntry with name {} to {}",
74-
entry.getName(),
75-
entryTargetPath);
76-
if (entry.isDirectory()) {
73+
String restOfPath = trimSlashes(entry.getName().substring(projectDirInZip.length()));
74+
75+
Path entryTargetPath = target.resolve(trimSlashes(restOfPath.replace("/", File.separator)));
76+
pathsInZip.add(entryTargetPath);
77+
78+
log.debug("Processing zipEntry with name {} to {}", entry.getName(), entryTargetPath);
79+
if (entry.isDirectory() || entryTargetPath.toFile().isDirectory()) {
7780
Files.createDirectories(entryTargetPath);
78-
} else {
79-
if (allowedToUnzip(entryTargetPath, target)) {
80-
log.trace("Allowed to unzip, unzipping");
81-
InputStream entryContent = zipFile.getInputStream(entry);
82-
FileUtils.copyInputStreamToFile(entryContent, entryTargetPath.toFile());
81+
log.debug("{} is a directory - creating and off to the next file ", entry.getName());
82+
continue;
83+
}
84+
boolean shouldWrite;
85+
InputStream entryContent = zipFile.getInputStream(entry);
86+
if (Files.exists(entryTargetPath)) {
87+
log.trace("Allowed to unzip, unzipping");
88+
byte[] entryData = IOUtils.toByteArray(entryContent);
89+
if (fileContentEquals(target.toFile(), entryData)) {
90+
shouldWrite = false;
91+
result.unchangedFiles.add(entryTargetPath);
92+
} else if (allowedToUnzip(entryTargetPath, target)) {
93+
shouldWrite = true;
94+
result.overwrittenFiles.add(entryTargetPath);
8395
} else {
84-
log.trace("Not allowed to unzip, skipping file");
96+
shouldWrite = false;
97+
result.skippedFiles.add(entryTargetPath);
8598
}
99+
} else {
100+
shouldWrite = true;
101+
result.newFiles.add(entryTargetPath);
102+
}
103+
if (shouldWrite) {
104+
FileUtils.copyInputStreamToFile(entryContent, entryTargetPath.toFile());
105+
} else {
106+
log.trace("Not allowed to unzip, skipping file");
107+
result.skippedFiles.add(entryTargetPath);
86108
}
87-
88109
log.debug("Done with file {}", entryTargetPath);
110+
89111
} else {
90112
log.debug("Skipping non project file from zip - {}", entry.getName());
91113
}
92114
}
93115
}
116+
94117
log.debug("Done unzipping");
118+
deleteFilesNotInZip(target, target, result, pathsInZip);
119+
return null;
120+
}
121+
122+
// TODO: validate
123+
private void deleteFilesNotInZip(Path projectDir, Path curDir, UnzipResult result, Set<Path> pathsInZip) throws IOException {
124+
125+
for (File file : curDir.toFile().listFiles()) {
126+
// Path relPath = Paths.get(trimSlashes(file.getPath().substring(projectDir.getPath().length())));
127+
Path filePath = file.toPath();
128+
if (file.isDirectory()) {
129+
deleteFilesNotInZip(projectDir, file.toPath(), result, pathsInZip);
130+
}
131+
132+
if (!pathsInZip.contains(filePath)) {
133+
if (mayDelete(filePath, null)) {
134+
if (file.isDirectory() && file.listFiles().length > 0) {
135+
// Won't delete directories if they still have contents
136+
result.skippedDeletingFiles.add(filePath);
137+
} else {
138+
file.delete();
139+
result.deletedFiles.add(filePath);
140+
}
141+
} else {
142+
result.skippedDeletingFiles.add(filePath);
143+
}
144+
}
145+
}
146+
}
147+
148+
private boolean fileContentEquals(File file, byte[] data) throws IOException {
149+
if (file.isDirectory()) {
150+
return false;
151+
}
152+
InputStream fileIs = new BufferedInputStream(new FileInputStream(file));
153+
InputStream dataIs = new ByteArrayInputStream(data);
154+
boolean eq = IOUtils.contentEquals(fileIs, dataIs);
155+
fileIs.close();
156+
dataIs.close();
157+
return eq;
95158
}
96159

97160
private String findProjectDirInZip(Enumeration<ZipArchiveEntry> zipEntries) throws IOException {
98-
ZipEntry zent;
99161
while (zipEntries.hasMoreElements()) {
100162
ZipArchiveEntry element = zipEntries.nextElement();
101163
String name = element.getName();
@@ -108,7 +170,7 @@ private String findProjectDirInZip(Enumeration<ZipArchiveEntry> zipEntries) thro
108170
return dirname(name);
109171
}
110172
}
111-
return null;
173+
throw new RuntimeException("No project in zip");
112174
}
113175

114176
private String dirname(String zipPath) {
@@ -145,4 +207,19 @@ private boolean allowedToUnzip(Path file, Path projectRoot) {
145207

146208
return true;
147209
}
210+
211+
private boolean mayDelete(Path file, Path projectRoot) {
212+
if (!Files.exists(file)) {
213+
log.trace("File does not exist, don't delete it");
214+
return false;
215+
}
216+
217+
log.trace("File exists, checking whether it's studentfile is allowed");
218+
219+
if (filePolicy.mayDelete(file, projectRoot)) {
220+
log.trace("File {} can be deleted", file);
221+
return true;
222+
}
223+
return false;
224+
}
148225
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/zip/UnzipResult.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package fi.helsinki.cs.tmc.langs.io.zip;
2+
3+
import java.nio.file.Path;
4+
import java.util.ArrayList;
5+
import java.util.List;
6+
7+
public class UnzipResult {
8+
/**
9+
* The project directory to which we extracted.
10+
*/
11+
public Path projectDir;
12+
13+
/**
14+
* Files that were in the zip but did not exist before.
15+
* In the usual case of downloading a new project, all files go here.
16+
*/
17+
public List<Path> newFiles = new ArrayList<>();
18+
19+
/**
20+
* Files overwritten as permitted by the given {@code OverwritingDecider}.
21+
*/
22+
public List<Path> overwrittenFiles = new ArrayList<>();
23+
24+
/**
25+
* Files skipped because the given {@code OverwritingDecider} didn't allow overwriting.
26+
*/
27+
public List<Path> skippedFiles = new ArrayList<>();
28+
29+
/**
30+
* Files that existed before but were the same in the zip.
31+
*/
32+
public List<Path> unchangedFiles = new ArrayList<>();
33+
34+
/**
35+
* Files that were deleted because they weren't in the zip.
36+
*/
37+
public List<Path> deletedFiles = new ArrayList<>();
38+
39+
/**
40+
* Files skipped because the given {@code OverwritingDecider} didn't allow deleting.
41+
*/
42+
public List<Path> skippedDeletingFiles = new ArrayList<>();
43+
44+
UnzipResult(Path projectDir) {
45+
this.projectDir = projectDir;
46+
}
47+
48+
@Override
49+
public String toString() {
50+
StringBuilder sb = new StringBuilder();
51+
sb.append("New: ").append(newFiles).append('\n');
52+
sb.append("Overwritten: ").append(overwrittenFiles).append('\n');
53+
sb.append("Skipped: ").append(skippedFiles).append('\n');
54+
sb.append("Unchanged: ").append(unchangedFiles).append('\n');
55+
sb.append("Deleted: ").append(deletedFiles).append('\n');
56+
sb.append("Not deleted: ").append(deletedFiles).append('\n');
57+
return sb.toString();
58+
}
59+
}

‎tmc-langs-framework/src/main/java/fi/helsinki/cs/tmc/langs/io/zip/Unzipper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
public interface Unzipper {
99

10-
void unzip(Path zipfile, Path target) throws IOException;
10+
UnzipResult unzip(Path zipFile, Path target) throws IOException;
1111

1212
void setStudentFilePolicy(StudentFilePolicy studentFilePolicy);
1313
}

‎tmc-langs-framework/src/test/java/fi/helsinki/cs/tmc/langs/io/zip/StudentFileAwareUnzipperTest.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,11 @@ private StudentFilePolicy getNothingIsStudentFilePolicy() {
135135
public boolean isStudentFile(Path path, Path projectRootPath) {
136136
return false;
137137
}
138+
139+
@Override
140+
public boolean mayDelete(Path file, Path projectRoot) {
141+
return true;
142+
}
138143
};
139144
}
140145

@@ -144,6 +149,11 @@ private StudentFilePolicy getEverythingIsStudentFilePolicy() {
144149
public boolean isStudentFile(Path path, Path projectRootPath) {
145150
return true;
146151
}
152+
153+
@Override
154+
public boolean mayDelete(Path file, Path projectRoot) {
155+
return false;
156+
}
147157
};
148158
}
149159
}

0 commit comments

Comments
 (0)