Add robots.txt and deny endpoints

Also add missing endpoint documentation.

Author: gaul

README.md

@@ -22,11 +22,15 @@ Java httpbin supports a subset of httpbin endpoints:
 - `/put`  Returns PUT data.
 - `/status/:code` Returns given HTTP Status code.
 - `/redirect/:n` 302 Redirects _n_ times.
+- `/absolute-redirect/:n` 302 Absolute redirects _n_ times.
 - `/redirect-to?url=foo` 302 Redirects to the _foo_ URL.
 - `/cookies` Returns the cookies.
 - `/cookies/set?name=value` Sets one or more simple cookies.
 - `/cookies/delete?name` Deletes one or more simple cookies.
+- `/robots.txt` Returns some robots.txt rules.
+- `/deny` Denied by robots.txt file.
 - `/basic-auth/:user/:passwd` Challenges HTTP Basic Auth.
+- `/hidden-basic-auth/:user/:passwd` Challenges HTTP Basic Auth and returns 404 on failure.
 - `/html` Returns some HTML.
 - `/xml` Returns some XML.
 - `/image/png` Returns page containing a PNG image.

src/main/java/org/gaul/httpbin/HttpBinHandler.java

@@ -275,6 +275,34 @@ public void handle(String target, Request baseRequest,
                 copyResource(servletResponse, "/text.xml");
                 baseRequest.setHandled(true);
                 return;
+            } else if (method.equals("GET") && uri.equals("/robots.txt")) {
+                byte[] output = "User-agent: *\nDisallow: /deny\n".getBytes(
+                        StandardCharsets.UTF_8);
+
+                servletResponse.setStatus(HttpServletResponse.SC_OK);
+                servletResponse.setContentType("text/plain");
+                os.write(output);
+                baseRequest.setHandled(true);
+                return;
+            } else if (method.equals("GET") && uri.equals("/deny")) {
+                byte[] output = (
+                        "    .-''''''-." +
+                        "  .' _      _ '." +
+                        " /   O      O   \"" +
+                        ":                :" +
+                        "|                |" +
+                        ":       __       :" +
+                        " \\  .-\"'  '\"-.  /" +
+                        "  '.          .'" +
+                        "     '-......-'" +
+                        "YOU SHOULDN'T BE HERE").getBytes(
+                                StandardCharsets.UTF_8);
+
+                servletResponse.setStatus(HttpServletResponse.SC_OK);
+                servletResponse.setContentType("text/plain");
+                os.write(output);
+                baseRequest.setHandled(true);
+                return;
             }
             servletResponse.setStatus(501);
             baseRequest.setHandled(true);