Skip to content

Commit 6efa928

Browse files
simschlasimschla
committed
1480: add caching around npm install
1 parent c2bba79 commit 6efa928

File tree

11 files changed

+236
-41
lines changed

11 files changed

+236
-41
lines changed

‎lib/src/main/java/com/diffplug/spotless/TimedLogger.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,9 @@ private String durationString() {
126126
if (duration < 1000) {
127127
return duration + "ms";
128128
} else if (duration < 1000 * 60) {
129-
return (duration / 1000) + "s";
129+
long seconds = duration / 1000;
130+
long millis = duration - seconds * 1000;
131+
return seconds + "." + millis + "s";
130132
} else {
131133
// output in the format 3m 4.321s
132134
long minutes = duration / (1000 * 60);

‎lib/src/main/java/com/diffplug/spotless/npm/NodeModulesCachingNpmProcessFactory.java

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright 2023 DiffPlug
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.diffplug.spotless.npm;
17+
18+
import java.io.File;
19+
import java.util.List;
20+
21+
import org.slf4j.Logger;
22+
import org.slf4j.LoggerFactory;
23+
24+
import com.diffplug.spotless.ProcessRunner.Result;
25+
import com.diffplug.spotless.TimedLogger;
26+
27+
public class NodeModulesCachingNpmProcessFactory implements NpmProcessFactory {
28+
29+
private static final Logger logger = LoggerFactory.getLogger(NodeModulesCachingNpmProcessFactory.class);
30+
31+
private static final TimedLogger timedLogger = TimedLogger.forLogger(logger);
32+
33+
private final File cacheDir;
34+
35+
private final ShadowCopy shadowCopy;
36+
37+
private NodeModulesCachingNpmProcessFactory(File cacheDir) {
38+
this.cacheDir = cacheDir;
39+
assertDir(cacheDir);
40+
this.shadowCopy = new ShadowCopy(cacheDir);
41+
}
42+
43+
private void assertDir(File cacheDir) {
44+
if (cacheDir.exists() && !cacheDir.isDirectory()) {
45+
throw new IllegalArgumentException("Cache dir must be a directory");
46+
}
47+
if (!cacheDir.exists()) {
48+
if (!cacheDir.mkdirs()) {
49+
throw new IllegalArgumentException("Cache dir could not be created.");
50+
}
51+
}
52+
}
53+
54+
public static NodeModulesCachingNpmProcessFactory forCacheDir(File cacheDir) {
55+
return new NodeModulesCachingNpmProcessFactory(cacheDir);
56+
}
57+
58+
@Override
59+
public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
60+
NpmProcess actualNpmInstallProcess = StandardNpmProcessFactory.INSTANCE.createNpmInstallProcess(nodeServerLayout, formatterStepLocations);
61+
return new CachingNmpInstall(actualNpmInstallProcess, nodeServerLayout);
62+
}
63+
64+
@Override
65+
public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
66+
return StandardNpmProcessFactory.INSTANCE.createNpmServeProcess(nodeServerLayout, formatterStepLocations);
67+
}
68+
69+
private class CachingNmpInstall implements NpmProcess {
70+
71+
private final NpmProcess actualNpmInstallProcess;
72+
private final NodeServerLayout nodeServerLayout;
73+
74+
public CachingNmpInstall(NpmProcess actualNpmInstallProcess, NodeServerLayout nodeServerLayout) {
75+
this.actualNpmInstallProcess = actualNpmInstallProcess;
76+
this.nodeServerLayout = nodeServerLayout;
77+
}
78+
79+
@Override
80+
public Result waitFor() {
81+
String entryName = entryName();
82+
if (shadowCopy.entryExists(entryName, NodeServerLayout.NODE_MODULES)) {
83+
timedLogger.withInfo("Using cached node_modules for {} from {}", entryName, cacheDir)
84+
.run(() -> shadowCopy.copyEntryInto(entryName(), NodeServerLayout.NODE_MODULES, nodeServerLayout.nodeModulesDir()));
85+
return new CachedResult();
86+
} else {
87+
Result result = actualNpmInstallProcess.waitFor();
88+
assert result.exitCode() == 0;
89+
// TODO: maybe spawn a thread to do this in the background?
90+
timedLogger.withInfo("Caching node_modules for {} in {}", entryName, cacheDir)
91+
.run(() -> shadowCopy.addEntry(entryName(), new File(nodeServerLayout.nodeModulesDir(), NodeServerLayout.NODE_MODULES)));
92+
return result;
93+
}
94+
}
95+
96+
private String entryName() {
97+
return nodeServerLayout.nodeModulesDir().getName();
98+
}
99+
100+
@Override
101+
public String describe() {
102+
return String.format("Wrapper around [%s] to cache node_modules in [%s]", actualNpmInstallProcess.describe(), cacheDir.getAbsolutePath());
103+
}
104+
}
105+
106+
private class CachedResult extends Result {
107+
108+
public CachedResult() {
109+
super(List.of("(from cache dir " + cacheDir + ")"), 0, new byte[0], new byte[0]);
110+
}
111+
}
112+
}

‎lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
class NodeServerLayout {
2828

2929
private static final Pattern PACKAGE_JSON_NAME_PATTERN = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\"");
30+
static final String NODE_MODULES = "node_modules";
3031

3132
private final File nodeModulesDir;
3233
private final File packageJsonFile;
@@ -55,7 +56,6 @@ private static String nodeModulesDirName(String packageJsonContent) {
5556
}
5657

5758
File nodeModulesDir() {
58-
5959
return nodeModulesDir;
6060
}
6161

@@ -89,7 +89,7 @@ public boolean isLayoutPrepared() {
8989
}
9090

9191
public boolean isNodeModulesPrepared() {
92-
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), "node_modules").toPath();
92+
Path nodeModulesInstallDirPath = new File(nodeModulesDir(), NODE_MODULES).toPath();
9393
if (!Files.isDirectory(nodeModulesInstallDirPath)) {
9494
return false;
9595
}

‎lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepStateBase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ protected NpmFormatterStepStateBase(String stepName, NpmConfig npmConfig, NpmFor
6262
this.npmConfig = requireNonNull(npmConfig);
6363
this.locations = locations;
6464
this.nodeServerLayout = new NodeServerLayout(locations.buildDir(), npmConfig.getPackageJsonContent());
65-
this.nodeServeApp = new NodeServeApp(nodeServerLayout, npmConfig, new StandardNpmProcessFactory(), locations);
65+
this.nodeServeApp = new NodeServeApp(nodeServerLayout, npmConfig, NodeModulesCachingNpmProcessFactory.forCacheDir(new File(locations.buildDir(), "spotless-npm-cache"))/*StandardNpmProcessFactory.INSTANCE*/, locations);
6666
}
6767

6868
protected void prepareNodeServerLayout() throws IOException {

‎lib/src/main/java/com/diffplug/spotless/npm/NpmLongRunningProcess.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright 2023 DiffPlug
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.diffplug.spotless.npm;
17+
18+
import com.diffplug.spotless.ProcessRunner.LongRunningProcess;
19+
20+
interface NpmLongRunningProcess {
21+
22+
String describe();
23+
24+
LongRunningProcess start();
25+
26+
}

‎lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,12 @@
1515
*/
1616
package com.diffplug.spotless.npm;
1717

18-
import java.util.concurrent.ExecutionException;
19-
20-
import com.diffplug.spotless.ProcessRunner.LongRunningProcess;
2118
import com.diffplug.spotless.ProcessRunner.Result;
2219

2320
interface NpmProcess {
2421

2522
String describe();
2623

27-
LongRunningProcess start();
28-
29-
default Result waitFor() {
30-
try (LongRunningProcess npmProcess = start()) {
31-
if (npmProcess.waitFor() != 0) {
32-
throw new NpmProcessException("Running npm command '" + describe() + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result());
33-
}
34-
return npmProcess.result();
35-
} catch (InterruptedException e) {
36-
throw new NpmProcessException("Running npm command '" + describe() + "' was interrupted.", e);
37-
} catch (ExecutionException e) {
38-
throw new NpmProcessException("Running npm command '" + describe() + "' failed.", e);
39-
}
40-
}
41-
42-
class NpmProcessException extends RuntimeException {
43-
private static final long serialVersionUID = 6424331316676759525L;
44-
45-
public NpmProcessException(String message) {
46-
super(message);
47-
}
24+
Result waitFor();
4825

49-
public NpmProcessException(String message, Throwable cause) {
50-
super(message, cause);
51-
}
52-
}
5326
}

‎lib/src/main/java/com/diffplug/spotless/npm/NpmProcessException.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright 2023 DiffPlug
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.diffplug.spotless.npm;
17+
18+
public class NpmProcessException extends RuntimeException {
19+
private static final long serialVersionUID = 6424331316676759525L;
20+
21+
public NpmProcessException(String message) {
22+
super(message);
23+
}
24+
25+
public NpmProcessException(String message, Throwable cause) {
26+
super(message, cause);
27+
}
28+
}

‎lib/src/main/java/com/diffplug/spotless/npm/NpmProcessFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
public interface NpmProcessFactory {
1919
NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);
2020

21-
NpmProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);
21+
NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations);
2222

2323
default String describe() {
2424
return getClass().getSimpleName();

‎lib/src/main/java/com/diffplug/spotless/npm/ShadowCopy.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ public File copyEntryInto(String key, String origName, File targetParentFolder)
116116
return target;
117117
}
118118

119+
public boolean entryExists(String key, String origName) {
120+
return entry(key, origName).exists();
121+
}
122+
119123
private static class CopyDirectoryRecursively extends SimpleFileVisitor<Path> {
120124
private final File target;
121125
private final File orig;

‎lib/src/main/java/com/diffplug/spotless/npm/StandardNpmProcessFactory.java

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,29 @@
1919
import java.io.IOException;
2020
import java.util.List;
2121
import java.util.Map;
22+
import java.util.concurrent.ExecutionException;
2223

2324
import com.diffplug.spotless.ProcessRunner;
2425

2526
public class StandardNpmProcessFactory implements NpmProcessFactory {
27+
28+
public static final StandardNpmProcessFactory INSTANCE = new StandardNpmProcessFactory();
29+
30+
private StandardNpmProcessFactory() {
31+
// only one instance neeeded
32+
}
33+
2634
@Override
2735
public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
2836
return new NpmInstall(nodeServerLayout.nodeModulesDir(), formatterStepLocations);
2937
}
3038

3139
@Override
32-
public NpmProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
40+
public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) {
3341
return new NpmServe(nodeServerLayout.nodeModulesDir(), formatterStepLocations);
3442
}
3543

36-
private static abstract class AbstractStandardNpmProcess implements NpmProcess {
44+
private static abstract class AbstractStandardNpmProcess {
3745
protected final ProcessRunner processRunner = ProcessRunner.usingRingBuffersOfCapacity(100 * 1024); // 100kB
3846

3947
protected final File workingDir;
@@ -55,22 +63,22 @@ protected Map<String, String> environmentVariables() {
5563
"PATH", formatterStepLocations.nodeExecutable().getParentFile().getAbsolutePath() + File.pathSeparator + System.getenv("PATH"));
5664
}
5765

58-
@Override
59-
public ProcessRunner.LongRunningProcess start() {
66+
protected ProcessRunner.LongRunningProcess doStart() {
6067
try {
6168
return processRunner.start(workingDir, environmentVariables(), null, true, commandLine());
6269
} catch (IOException e) {
6370
throw new NpmProcessException("Failed to launch npm command '" + describe() + "'.", e);
6471
}
6572
}
6673

67-
@Override
68-
public String describe() {
74+
protected abstract String describe();
75+
76+
public String doDescribe() {
6977
return String.format("%s in %s [%s]", getClass().getSimpleName(), workingDir, String.join(" ", commandLine()));
7078
}
7179
}
7280

73-
private static class NpmInstall extends AbstractStandardNpmProcess {
81+
private static class NpmInstall extends AbstractStandardNpmProcess implements NpmProcess {
7482

7583
public NpmInstall(File workingDir, NpmFormatterStepLocations formatterStepLocations) {
7684
super(workingDir, formatterStepLocations);
@@ -85,9 +93,28 @@ protected List<String> commandLine() {
8593
"--no-fund",
8694
"--prefer-offline");
8795
}
96+
97+
@Override
98+
public String describe() {
99+
return doDescribe();
100+
}
101+
102+
@Override
103+
public ProcessRunner.Result waitFor() {
104+
try (ProcessRunner.LongRunningProcess npmProcess = doStart()) {
105+
if (npmProcess.waitFor() != 0) {
106+
throw new NpmProcessException("Running npm command '" + describe() + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result());
107+
}
108+
return npmProcess.result();
109+
} catch (InterruptedException e) {
110+
throw new NpmProcessException("Running npm command '" + describe() + "' was interrupted.", e);
111+
} catch (ExecutionException e) {
112+
throw new NpmProcessException("Running npm command '" + describe() + "' failed.", e);
113+
}
114+
}
88115
}
89116

90-
private static class NpmServe extends AbstractStandardNpmProcess {
117+
private static class NpmServe extends AbstractStandardNpmProcess implements NpmLongRunningProcess {
91118

92119
public NpmServe(File workingDir, NpmFormatterStepLocations formatterStepLocations) {
93120
super(workingDir, formatterStepLocations);
@@ -100,5 +127,15 @@ protected List<String> commandLine() {
100127
"start",
101128
"--scripts-prepend-node-path=true");
102129
}
130+
131+
@Override
132+
public String describe() {
133+
return doDescribe();
134+
}
135+
136+
@Override
137+
public ProcessRunner.LongRunningProcess start() {
138+
return doStart();
139+
}
103140
}
104141
}

0 commit comments

Comments
 (0)