Skip to content

Commit 77d9fd2

Browse files
vjkoskelaBrandonArp
vjkoskela
authored and
BrandonArp
committed
Http source (#82)
* Create an HTTP source * Http source fixes (#80) * Added http status metrics to match mad. * Support ISO8601 for KairosDbSink histogramTtl and ttl parameters. (#79) * Add conversion to PeriodicData to HttpSource.
1 parent 2a57d7f commit 77d9fd2

File tree

8 files changed

+566
-55
lines changed

8 files changed

+566
-55
lines changed

‎jdk-wrapper.sh‎

Lines changed: 106 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -40,45 +40,109 @@ safe_command() {
4040
fi
4141
}
4242

43-
download() {
43+
checksum() {
44+
l_file="$1"
45+
checksum_exec=""
46+
if command -v sha256sum > /dev/null; then
47+
checksum_exec="sha256sum"
48+
elif command -v shasum > /dev/null; then
49+
checksum_exec="shasum -a 256"
50+
elif command -v sha1sum > /dev/null; then
51+
checksum_exec="sha1sum"
52+
elif command -v md5 > /dev/null; then
53+
checksum_exec="md5"
54+
fi
55+
if [ -z "${checksum_exec}" ]; then
56+
log_err "ERROR: No supported checksum command found!"
57+
exit 1
58+
fi
59+
cat "${l_file}" | ${checksum_exec}
60+
}
61+
62+
rand() {
63+
awk 'BEGIN {srand();printf "%d\n", (rand() * 10^8);}'
64+
}
65+
66+
download_if_needed() {
4467
file="$1"
45-
if [ ! -f "${JDKW_PATH}/${file}" ]; then
46-
jdkw_url="${JDKW_URI}/${file}"
68+
path="$2"
69+
if [ ! -f "${path}/${file}" ]; then
70+
jdkw_url="${JDKW_BASE_URI}/releases/download/${JDKW_RELEASE}/${file}"
4771
log_out "Downloading ${file} from ${jdkw_url}"
48-
safe_command "curl ${CURL_OPTIONS} -f -k -L -o \"${JDKW_PATH}/${file}\" \"${jdkw_url}\""
49-
safe_command "chmod +x \"${JDKW_PATH}/${file}\""
72+
safe_command "curl ${curl_options} -f -k -L -o \"${path}/${file}\" \"${jdkw_url}\""
73+
safe_command "chmod +x \"${path}/${file}\""
5074
fi
5175
}
5276

5377
# Default curl options
54-
CURL_OPTIONS=""
78+
curl_options=""
79+
80+
# Process (but do not load) properties from environment
81+
env_configuration=
82+
l_fifo="${TMPDIR:-/tmp}/$$.$(rand)"
83+
safe_command "mkfifo \"${l_fifo}\""
84+
env > "${l_fifo}" &
85+
while IFS='=' read -r name value
86+
do
87+
jdkw_arg=$(echo "${name}" | grep '^JDKW_.*')
88+
jdkw_base_dir_arg=$(echo "${name}" | grep '^JDKW_BASE_DIR')
89+
if [ -n "${jdkw_base_dir_arg}" ]; then
90+
eval "${name}=\"${value}\""
91+
fi
92+
if [ -n "${jdkw_arg}" ]; then
93+
env_configuration="${env_configuration}${name}=\"${value}\" "
94+
fi
95+
done < "${l_fifo}"
96+
safe_command "rm \"${l_fifo}\""
97+
98+
# Process (but do not load) properties from command line arguments
99+
command=
100+
cmd_configuration=
101+
for arg in "$@"; do
102+
if [ -z ${in_command} ]; then
103+
jdkw_arg=$(echo "${arg}" | grep '^JDKW_.*')
104+
jdkw_base_dir_arg=$(echo "${arg}" | grep '^JDKW_BASE_DIR.*')
105+
if [ -n "${jdkw_base_dir_arg}" ]; then
106+
eval ${arg}
107+
fi
108+
if [ -n "${jdkw_arg}" ]; then
109+
cmd_configuration="${cmd_configuration}${arg} "
110+
fi
111+
fi
112+
case "${arg}" in
113+
*\'*)
114+
arg=`printf "%s" "$arg" | sed "s/'/'\"'\"'/g"`
115+
;;
116+
*) : ;;
117+
esac
118+
command="${command} '${arg}'"
119+
done
120+
121+
# Default base directory to current working directory
122+
if [ -z "${JDKW_BASE_DIR}" ]; then
123+
JDKW_BASE_DIR="."
124+
fi
55125

56126
# Load properties file in home directory
57127
if [ -f "${HOME}/.jdkw" ]; then
58128
. "${HOME}/.jdkw"
59129
fi
60130

61-
# Load properties file in working directory
62-
if [ -f ".jdkw" ]; then
63-
. "./.jdkw"
131+
# Load properties file in base directory
132+
if [ -f "${JDKW_BASE_DIR}/.jdkw" ]; then
133+
. "${JDKW_BASE_DIR}/.jdkw"
64134
fi
65135

66-
# Process command line arguments
67-
for ARG in "$@"; do
68-
JDKW_ARG=$(echo "${ARG}" | grep 'JDKW_.*')
69-
if [ -n "${JDKW_ARG}" ]; then
70-
eval ${ARG}
71-
else
72-
break
73-
fi
74-
done
136+
# Load properties from environment
137+
eval "${env_configuration}"
75138

76-
# Globals
77-
JDKW_BASE_URI="https://github.com/KoskiLabs/jdk-wrapper"
78-
JDKW_IMPL="jdkw-impl.sh"
79-
JDKW_WRAPPER="jdk-wrapper.sh"
139+
# Load properties from command line arguments
140+
eval "${cmd_configuration}"
80141

81142
# Process configuration
143+
if [ -z "${JDKW_BASE_URI}" ]; then
144+
JDKW_BASE_URI="https://github.com/KoskiLabs/jdk-wrapper"
145+
fi
82146
if [ -z "${JDKW_RELEASE}" ]; then
83147
JDKW_RELEASE="latest"
84148
log_out "Defaulted to version ${JDKW_RELEASE}"
@@ -88,38 +152,42 @@ if [ -z "${JDKW_TARGET}" ]; then
88152
log_out "Defaulted to target ${JDKW_TARGET}"
89153
fi
90154
if [ -z "${JDKW_VERBOSE}" ]; then
91-
CURL_OPTIONS="${CURL_OPTIONS} --silent"
155+
curl_options="${curl_options} --silent"
92156
fi
93157

94158
# Resolve latest version
95159
if [ "${JDKW_RELEASE}" = "latest" ]; then
96-
JDKW_RELEASE=$(curl ${CURL_OPTIONS} -f -k -L -H 'Accept: application/json' "${JDKW_BASE_URI}/releases/latest" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
160+
latest_version_json="${TMPDIR:-/tmp}/jdkw-latest-version-$$.$(rand)"
161+
safe_command "curl ${curl_options} -f -k -L -o \"${latest_version_json}\" -H 'Accept: application/json' \"${JDKW_BASE_URI}/releases/latest\""
162+
JDKW_RELEASE=$(cat "${latest_version_json}" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
163+
rm -f "${latest_version_json}"
97164
log_out "Resolved latest version to ${JDKW_RELEASE}"
98165
fi
99166

100-
# Define source and target
101-
JDKW_URI="${JDKW_BASE_URI}/releases/download/${JDKW_RELEASE}"
102-
JDKW_PATH="${JDKW_TARGET}/jdkw/${JDKW_RELEASE}"
103-
104167
# Ensure target directory exists
105-
if [ ! -d "${JDKW_PATH}" ]; then
106-
log_out "Creating target directory ${JDKW_PATH}"
107-
safe_command "mkdir -p \"${JDKW_PATH}\""
168+
jdkw_path="${JDKW_TARGET}/jdkw/${JDKW_RELEASE}"
169+
if [ ! -d "${jdkw_path}" ]; then
170+
log_out "Creating target directory ${jdkw_path}"
171+
safe_command "mkdir -p \"${jdkw_path}\""
108172
fi
109173

110174
# Download the jdk wrapper version
111-
download "${JDKW_IMPL}"
112-
download "${JDKW_WRAPPER}"
175+
jdkw_impl="jdkw-impl.sh"
176+
jdkw_wrapper="jdk-wrapper.sh"
177+
download_if_needed "${jdkw_impl}" "${jdkw_path}"
178+
download_if_needed "${jdkw_wrapper}" "${jdkw_path}"
179+
180+
# Execute the provided command
181+
eval ${jdkw_path}/${jdkw_impl} ${command}
182+
result=$?
113183

114184
# Check whether this wrapper is the one specified for this version
115-
jdkw_download="${JDKW_PATH}/${JDKW_WRAPPER}"
185+
jdkw_download="${jdkw_path}/${jdkw_wrapper}"
116186
jdkw_current="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/$(basename "$0")"
117-
if [ "$(cat "${jdkw_download}" | sha1sum )" != "$(cat "${jdkw_current}" | sha1sum)" ]; then
187+
if [ "$(checksum "${jdkw_download}")" != "$(checksum "${jdkw_current}")" ]; then
118188
printf "\e[0;31m[WARNING]\e[0m Your jdk-wrapper.sh file does not match the one in your JDKW_RELEASE.\n"
119189
printf "\e[0;32mUpdate your jdk-wrapper.sh to match by running:\e[0m\n"
120190
printf "cp \"%s\" \"%s\"\n" "${jdkw_download}" "${jdkw_current}"
121191
fi
122192

123-
# Execute the provided command
124-
${JDKW_PATH}/${JDKW_IMPL} $@
125-
exit $?
193+
exit ${result}

‎maven/maven-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#Maven download properties
2-
#Fri Dec 08 05:16:17 UTC 2017
2+
#Thu Feb 22 06:38:47 UTC 2018
33
checksumAlgorithm=SHA1
44
verifyDownload=true
5-
distributionUrl=https\://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip
5+
distributionUrl=https\://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip

‎pom.xml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<parent>
1919
<groupId>com.arpnetworking.build</groupId>
2020
<artifactId>arpnetworking-parent-pom</artifactId>
21-
<version>1.1.4</version>
21+
<version>1.1.5</version>
2222
<relativePath />
2323
</parent>
2424

@@ -108,7 +108,7 @@
108108
<apache.httpclient.version>4.5.2</apache.httpclient.version>
109109
<apache.httpcore.version>4.4.5</apache.httpcore.version>
110110
<arpnetworking.commons.version>1.13.3</arpnetworking.commons.version>
111-
<aspectjrt.version>1.8.10</aspectjrt.version>
111+
<aspectjrt.version>1.9.1</aspectjrt.version>
112112
<cglib.version>3.2.5</cglib.version>
113113
<commons.codec.version>1.10</commons.codec.version>
114114
<ebean.version>6.8.1</ebean.version>
@@ -134,7 +134,6 @@
134134
<logback.steno.version>1.18.0</logback.steno.version>
135135
<log4j.over.slf4j.version>1.7.22</log4j.over.slf4j.version>
136136
<luaj.version>3.0.1</luaj.version>
137-
<maven.version>3.3.9</maven.version>
138137
<metrics.client.version>0.10.0</metrics.client.version>
139138
<metrics.jvm.extra.version>0.9.0</metrics.jvm.extra.version>
140139
<metrics.http.extra.version>0.9.1</metrics.http.extra.version>
@@ -159,8 +158,8 @@
159158

160159
<!--Plugin versions-->
161160
<avaje.ebeanorm.mavenenhancer.version>4.7.1</avaje.ebeanorm.mavenenhancer.version>
162-
<aspectjtools.maven.plugin>1.8.9</aspectjtools.maven.plugin>
163-
<aspectj.maven.plugin.version>1.8</aspectj.maven.plugin.version>
161+
<aspectjtools.maven.plugin>1.9.1</aspectjtools.maven.plugin>
162+
<aspectj.maven.plugin.version>1.11</aspectj.maven.plugin.version>
164163
<docker.maven.plugin.version>0.15.9</docker.maven.plugin.version>
165164
<javassist.maven.plugin.version>0.1.2</javassist.maven.plugin.version>
166165
<maven.assembly.plugin.version>2.6</maven.assembly.plugin.version>

‎src/main/java/com/arpnetworking/clusteraggregator/GuiceModule.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.arpnetworking.clusteraggregator.bookkeeper.persistence.InMemoryBookkeeper;
4141
import com.arpnetworking.clusteraggregator.client.AggClientServer;
4242
import com.arpnetworking.clusteraggregator.client.AggClientSupervisor;
43+
import com.arpnetworking.clusteraggregator.client.HttpSourceActor;
4344
import com.arpnetworking.clusteraggregator.configuration.ClusterAggregatorConfiguration;
4445
import com.arpnetworking.clusteraggregator.configuration.ConfigurableActorProxy;
4546
import com.arpnetworking.clusteraggregator.configuration.DatabaseConfiguration;
@@ -112,6 +113,13 @@ protected void configure() {
112113
.toProvider(new DatabaseProvider(entry.getKey(), entry.getValue()))
113114
.in(Singleton.class);
114115
}
116+
117+
bind(String.class).annotatedWith(Names.named("health-check-path")).toInstance(_configuration.getHttpHealthCheckPath());
118+
bind(String.class).annotatedWith(Names.named("status-path")).toInstance(_configuration.getHttpStatusPath());
119+
bind(ActorRef.class)
120+
.annotatedWith(Names.named("http-ingest-v1"))
121+
.toProvider(GuiceActorCreator.provider(HttpSourceActor.class, "http-ingest-v1"))
122+
.asEagerSingleton();
115123
}
116124

117125
@Provides
@@ -258,15 +266,10 @@ private ActorRef provideAggregatorLifecycleTracker(
258266
@SuppressFBWarnings("UPM_UNCALLED_PRIVATE_METHOD") // Invoked reflectively by Guice
259267
private java.util.concurrent.CompletionStage<akka.http.javadsl.ServerBinding> provideHttpServer(
260268
final ActorSystem system,
261-
final MetricsFactory metricsFactory) {
269+
final Routes routes) {
262270

263271
// Create and bind Http server
264272
final Materializer materializer = ActorMaterializer.create(system);
265-
final Routes routes = new Routes(
266-
system,
267-
metricsFactory,
268-
_configuration.getHttpHealthCheckPath(),
269-
_configuration.getHttpStatusPath());
270273
final Http http = Http.get(system);
271274
final akka.stream.javadsl.Source<IncomingConnection, CompletionStage<ServerBinding>> binding = http.bind(
272275
ConnectHttp.toHost(

0 commit comments

Comments
 (0)