Merge pull request #128 from baranyaib90/quic_and_robot

Fixes 8: Quic and robot

Author: aarond10

.github/workflows/cmake.yml

@@ -21,7 +21,10 @@ jobs:
       run: sudo apt-get update
 
     - name: Setup Dependencies
-      run: sudo apt-get install cmake libc-ares-dev libcurl4-openssl-dev libev-dev build-essential clang-tidy-12 ${{ matrix.compiler }}
+      run: sudo apt-get install cmake libc-ares-dev libcurl4-openssl-dev libev-dev build-essential clang-tidy-12 ${{ matrix.compiler }} dnsutils python3-pip
+
+    - name: Setup Robot Framework
+      run: sudo pip3 install robotframework
 
     - name: Set clang-tidy
       run: sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-12 100
@@ -36,3 +39,12 @@ jobs:
         CC: ${{ matrix.compiler }}
       # Build your program with the given configuration
       run: make -C ${{github.workspace}}/
+
+    - name: Test
+      run: make -C ${{github.workspace}}/ test ARGS="--verbose"
+
+    - uses: actions/upload-artifact@v2
+      if: ${{ success() || failure() }}
+      with:
+        name: robot-logs-${{ matrix.compiler }}
+        path: ${{github.workspace}}/tests/robot/*.html

.gitignore

@@ -13,3 +13,6 @@ install_manifest.txt
 .ninja_log
 build.ninja
 rules.ninja
+log.html
+output.xml
+report.html

CMakeLists.txt

@@ -1,6 +1,8 @@
 project(HttpsDnsProxy C)
 cmake_minimum_required(VERSION 3.7)
 
+# FUNCTIONS
+
 # source: https://stackoverflow.com/a/27990434
 function(define_file_basename_for_sources targetname)
   get_target_property(source_files "${targetname}" SOURCES)
@@ -12,6 +14,8 @@ function(define_file_basename_for_sources targetname)
   endforeach()
 endfunction()
 
+# CONFIG
+
 set(CMAKE_BUILD_TYPE "Debug")
 #set(CMAKE_BUILD_TYPE "Release")
 
@@ -24,6 +28,8 @@ if ((CMAKE_C_COMPILER_ID MATCHES GNU   AND CMAKE_C_COMPILER_VERSION VERSION_GREA
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-gnu-zero-variadic-macro-arguments -Wno-gnu-folding-constant")
 endif()
 
+# VERSION
+
 find_package(Git)
 if(Git_FOUND)
   execute_process(
@@ -42,13 +48,17 @@ else()
   message(WARNING "Could not find git command! Version is set to: ${GIT_VERSION}")
 endif()
 
+# LIBRARY DEPENDENCIES
+
 find_path(LIBCARES_INCLUDE_DIR ares.h)
 find_path(LIBCURL_INCLUDE_DIR curl/curl.h)
 find_path(LIBEV_INCLUDE_DIR ev.h)
 include_directories(
   ${LIBCARES_INCLUDE_DIR} ${LIBCURL_INCLUDE_DIR}
   ${LIBEV_INCLUDE_DIR} src)
 
+# CLANG TIDY
+
 find_program(
   CLANG_TIDY_EXE
   NAMES "clang-tidy"
@@ -61,6 +71,8 @@ else()
   set(DO_CLANG_TIDY "${CLANG_TIDY_EXE}" "-fix" "-checks=*,-clang-analyzer-alpha.*,-misc-unused-parameters,-cert-err34-c,-google-readability-todo,-hicpp-signed-bitwise,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers,-gnu-folding-constant,-gnu-zero-variadic-macro-arguments,-readability-function-cognitive-complexity,-concurrency-mt-unsafe")
 endif()
 
+# BUILD
+
 # The main binary
 set(TARGET_NAME "https_dns_proxy")
 aux_source_directory(src SRC_LIST)
@@ -82,6 +94,8 @@ if(CLANG_TIDY_EXE)
   )
 endif()
 
+# INSTALL
+
 install(TARGETS ${TARGET_NAME} DESTINATION bin)
 
 set(SERVICE_EXTRA_OPTIONS "")
@@ -100,3 +114,20 @@ configure_file(${TARGET_NAME}.service.in ${TARGET_NAME}.service)
 
 install(FILES ${TARGET_NAME}.service
         DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/systemd/system/)
+
+# TESTING
+
+find_program(
+  PYTHON3_EXE
+  NAMES "python3"
+  DOC "Path to python3 executable"
+  )
+if(NOT PYTHON3_EXE)
+  message(STATUS "python3 not found, robot testing not possible")
+else()
+  message(STATUS "python3 found: ${PYTHON3_EXE}")
+
+  enable_testing()
+  add_test(NAME robot COMMAND ${PYTHON3_EXE} -m robot.run functional_tests.robot
+           WORKING_DIRECTORY tests/robot)
+endif()

README.md

@@ -146,9 +146,9 @@ Just run it as a daemon and point traffic at it. Commandline flags are:
 ```
 Usage: ./https_dns_proxy [-a <listen_addr>] [-p <listen_port>]
         [-d] [-u <user>] [-g <group>] [-b <dns_servers>]
-        [-r <resolver_url>] [-e <subnet_addr>]
-        [-t <proxy_server>] [-l <logfile>] -c <dscp_codepoint>
-        [-x] [-v]+
+        [-i <polling_interval>] [-4] [-r <resolver_url>]
+        [-t <proxy_server>] [-l <logfile>] [-c <dscp_codepoint>]
+        [-x] [-q] [-s <statistic_interval>] [-v]+ [-V] [-h]
 
   -a listen_addr         Local IPv4/v6 address to bind to. (127.0.0.1)
   -p listen_port         Local port to bind to. (5053)
@@ -162,7 +162,7 @@ Usage: ./https_dns_proxy [-a <listen_addr>] [-p <listen_port>]
   -i polling_interval    Optional polling interval of DNS servers.
                          (Default: 120, Min: 5, Max: 3600)
   -4                     Force IPv4 hostnames for DNS resolvers non IPv6 networks.
-  -r resolver_url        The HTTPS path to the resolver URL. default: https://dns.google/dns-query
+  -r resolver_url        The HTTPS path to the resolver URL. Default: https://dns.google/dns-query
   -t proxy_server        Optional HTTP proxy. e.g. socks5://127.0.0.1:1080
                          Remote name resolution will be used if the protocol
                          supports it (http, https, socks4a, socks5h), otherwise
@@ -173,18 +173,35 @@ Usage: ./https_dns_proxy [-a <listen_addr>] [-p <listen_port>]
                          connections.
   -x                     Use HTTP/1.1 instead of HTTP/2. Useful with broken
                          or limited builds of libcurl. (false)
+  -q                     Use HTTP/3 (QUIC) only. (false)
   -s statistic_interval  Optional statistic printout interval.
                          (Default: 0, Disabled: 0, Min: 1, Max: 3600)
-  -v                     Increase logging verbosity. (INFO)
+  -v                     Increase logging verbosity. (Default: error)
+                         Levels: fatal, stats, error, warning, info, debug
+                         Request issues are logged on warning level.
   -V                     Print version and exit.
+  -h                     Print help and exit.
+```
+
+## Testing
+
+Functional tests can be executed using [Robot Framework](https://robotframework.org/).
+
+dig command is expected to be available.
+
+```
+pip3 install robotframework
+python3 -m robot.run tests/robot/functional_tests.robot
 ```
 
 ## TODO
 
 * Add some tests.
+* Improve IPv6 handling and add automatic fallback to IPv4
 
 ## Authors
 
 * Aaron Drew ([email protected]): Original https_dns_proxy.
 * Soumya ([github.com/soumya92](https://github.com/soumya92)): RFC 8484 implementation.
+* baranyaib90 ([github.com/baranyaib90](https://github.com/baranyaib90)): fixes and improvements.
 

src/https_client.c

@@ -212,6 +212,60 @@ int https_curl_debug(CURL __attribute__((unused)) * handle, curl_infotype type,
   return 0;
 }
 
+static const char * http_version_str(const long version) {
+  switch (version) {
+    case CURL_HTTP_VERSION_1_0:
+      return "1.0";
+    case CURL_HTTP_VERSION_1_1:
+      return "1.1";
+    case CURL_HTTP_VERSION_2_0: // fallthrough
+    case CURL_HTTP_VERSION_2TLS:
+      return "2";
+#ifdef CURL_VERSION_HTTP3
+    case CURL_HTTP_VERSION_3:
+      return "3";
+#endif
+    default:
+      FLOG("Unsupported HTTP version: %d", version);
+  }
+  return "UNKNOWN"; // unreachable code
+}
+
+static void https_set_request_version(https_client_t *client,
+                                      struct https_fetch_ctx *ctx) {
+  long http_version_int = CURL_HTTP_VERSION_2TLS;
+  switch (client->opt->use_http_version) {
+    case 1:
+      http_version_int = CURL_HTTP_VERSION_1_1;
+      // fallthrough
+    case 2:
+      break;
+    case 3:
+#ifdef CURL_VERSION_HTTP3
+      http_version_int = CURL_HTTP_VERSION_3;
+#endif
+      break;
+    default:
+      FLOG_REQ("Invalid HTTP version: %d", client->opt->use_http_version);
+  }
+  DLOG_REQ("Requesting HTTP/%s", http_version_str(http_version_int));
+
+  CURLcode easy_code = curl_easy_setopt(ctx->curl, CURLOPT_HTTP_VERSION, http_version_int);
+  if (easy_code != CURLE_OK) {
+    ELOG_REQ("Setting HTTP/%s version failed with %d: %s",
+             http_version_str(http_version_int), easy_code, curl_easy_strerror(easy_code));
+
+    if (client->opt->use_http_version == 3) {
+      ELOG("Try to run application without -q argument!");  // fallback unknown for current request
+      client->opt->use_http_version = 2;
+    } else if (client->opt->use_http_version == 2) {
+      ELOG("Try to run application with -x argument! Falling back to HTTP/1.1 version.");
+      client->opt->use_http_version = 1;
+      // TODO: consider CURLMOPT_PIPELINING setting??
+    }
+  }
+}
+
 static void https_fetch_ctx_init(https_client_t *client,
                                  struct https_fetch_ctx *ctx, const char *url,
                                  const char* data, size_t datalen,
@@ -228,20 +282,7 @@ static void https_fetch_ctx_init(https_client_t *client,
 
   ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_RESOLVE, resolv);
 
-  DLOG_REQ("Requesting HTTP/1.1: %d", client->opt->use_http_1_1);
-  CURLcode easy_code = curl_easy_setopt(ctx->curl, CURLOPT_HTTP_VERSION,
-                                        client->opt->use_http_1_1 ?
-                                        CURL_HTTP_VERSION_1_1 :
-                                        CURL_HTTP_VERSION_2_0);
-  if (easy_code != CURLE_OK) {
-    // hopefully errors will be logged once
-    ELOG_REQ("CURLOPT_HTTP_VERSION error %d: %s",
-             easy_code, curl_easy_strerror(easy_code));
-    if (!client->opt->use_http_1_1) {
-      ELOG("Try to run application with -x argument! Forcing HTTP/1.1 version.");
-      client->opt->use_http_1_1 = 1;
-    }
-  }
+  https_set_request_version(client, ctx);
 
   if (logging_debug_enabled()) {
     ASSERT_CURL_EASY_SETOPT(ctx, CURLOPT_VERBOSE, 1L);
@@ -301,7 +342,7 @@ static int https_fetch_ctx_process_response(https_client_t *client,
       faulty_response = 0;
       break;
     case CURLE_WRITE_ERROR:
-      WLOG_REQ("Response content was too large");
+      WLOG_REQ("curl request failed with write error (probably response content was too large)");
       break;
     default:
       WLOG_REQ("curl request failed with %d: %s", res, curl_easy_strerror(res));
@@ -322,9 +363,10 @@ static int https_fetch_ctx_process_response(https_client_t *client,
           uploaded_bytes > 0) {
         WLOG_REQ("Connecting and sending request to resolver was successful, "
                  "but no response was sent back");
-        if (client->opt->use_http_1_1) {
+        if (client->opt->use_http_version == 1) {
           // for example Unbound DoH servers does not support HTTP/1.x, only HTTP/2
-          WLOG("Resolver may not support current HTTP/1.1 protocol version");
+          WLOG("Resolver may not support current HTTP/%s protocol version",
+               http_version_str(client->opt->use_http_version));
         }
       } else {
         // in case of HTTP/1.1 this can happen very often depending on DNS query frequency
@@ -404,20 +446,8 @@ static int https_fetch_ctx_process_response(https_client_t *client,
     if ((res = curl_easy_getinfo(
             ctx->curl, CURLINFO_HTTP_VERSION, &long_resp)) != CURLE_OK) {
       ELOG_REQ("CURLINFO_HTTP_VERSION: %s", curl_easy_strerror(res));
-    } else {
-      switch (long_resp) {
-        case CURL_HTTP_VERSION_1_0:
-          DLOG_REQ("CURLINFO_HTTP_VERSION: 1.0");
-          break;
-        case CURL_HTTP_VERSION_1_1:
-          DLOG_REQ("CURLINFO_HTTP_VERSION: 1.1");
-          break;
-        case CURL_HTTP_VERSION_2_0:
-          DLOG_REQ("CURLINFO_HTTP_VERSION: 2");
-          break;
-        default:
-          DLOG_REQ("CURLINFO_HTTP_VERSION: %d", long_resp);
-      }
+    } else if (long_resp != CURL_HTTP_VERSION_NONE) {
+      DLOG_REQ("CURLINFO_HTTP_VERSION: %s", http_version_str(long_resp));
     }
     if ((res = curl_easy_getinfo(
             ctx->curl, CURLINFO_PROTOCOL, &long_resp)) != CURLE_OK) {

src/main.c

@@ -226,6 +226,22 @@ int main(int argc, char *argv[]) {
   // through to errors about use of uninitialized values in our code. :(
   curl_global_init(CURL_GLOBAL_DEFAULT);
 
+  curl_version_info_data *curl_ver = curl_version_info(CURLVERSION_NOW);
+  if (!(curl_ver->features & CURL_VERSION_HTTP2)) {
+    WLOG("HTTP/2 is not supported by current libcurl");
+  }
+#ifdef CURL_VERSION_HTTP3
+  if (!(curl_ver->features & CURL_VERSION_HTTP3))
+  {
+    WLOG("HTTP/3 is not supported by current libcurl");
+  }
+#else
+  WLOG("HTTP/3 was not available at build time, it will not work at all");
+#endif
+  if (!(curl_ver->features & CURL_VERSION_IPV6)) {
+    WLOG("IPv6 is not supported by current libcurl");
+  }
+
   // Note: This calls ev_default_loop(0) which never cleans up.
   //       valgrind will report a leak. :(
   struct ev_loop *loop = EV_DEFAULT;

src/options.c

@@ -15,6 +15,8 @@
 #define O_CLOEXEC 0
 #endif
 
+#define DEFAULT_HTTP_VERSION 2
+
 void options_init(struct Options *opt) {
   opt->listen_addr = "127.0.0.1";
   opt->listen_port = 5053;
@@ -33,13 +35,13 @@ void options_init(struct Options *opt) {
   opt->ipv4 = 0;
   opt->resolver_url = "https://dns.google/dns-query";
   opt->curl_proxy = NULL;
-  opt->use_http_1_1 = 0;
+  opt->use_http_version = DEFAULT_HTTP_VERSION;
   opt->stats_interval = 0;
 }
 
 int options_parse_args(struct Options *opt, int argc, char **argv) {
   int c = 0;
-  while ((c = getopt(argc, argv, "a:c:p:du:g:b:i:4r:e:t:l:vxs:hV")) != -1) {
+  while ((c = getopt(argc, argv, "a:c:p:du:g:b:i:4r:e:t:l:vxqs:hV")) != -1) {
     switch (c) {
     case 'a': // listen_addr
       opt->listen_addr = optarg;
@@ -82,8 +84,15 @@ int options_parse_args(struct Options *opt, int argc, char **argv) {
         opt->loglevel--;
       }
       break;
-    case 'x': // http/1.1
-      opt->use_http_1_1 = 1;
+    case 'x': // http/1.1 fallthrough
+    case 'q': // http/3
+      if (opt->use_http_version == DEFAULT_HTTP_VERSION) {
+        opt->use_http_version = (c == 'x' ? 1 : 3);
+      } else {
+        printf("HTTP version already set to: HTTP/%s\n",
+               opt->use_http_version == 1 ? "1.1" : "3");
+        return -1;
+      }
       break;
     case 's': // stats interval
       opt->stats_interval = atoi(optarg);
@@ -193,6 +202,7 @@ void options_show_usage(int __attribute__((unused)) argc, char **argv) {
   printf("                         connections.\n");
   printf("  -x                     Use HTTP/1.1 instead of HTTP/2. Useful with broken\n"
          "                         or limited builds of libcurl. (false)\n");
+  printf("  -q                     Use HTTP/3 (QUIC) only. (false)\n");
   printf("  -s statistic_interval  Optional statistic printout interval.\n"\
          "                         (Default: %d, Disabled: 0, Min: 1, Max: 3600)\n",
          defaults.stats_interval);

src/options.h

@@ -41,8 +41,10 @@ struct Options {
   // e.g. "socks5://127.0.0.1:1080"
   const char *curl_proxy;
 
-  // Hack to fix OpenWRT issues due to dropping of HTTP/2 support from libcurl.
-  int use_http_1_1;
+  // 1 = Use only HTTP/1.1 for limited OpenWRT libcurl (which is not built with HTTP/2 support)
+  // 2 = Use only HTTP/2 default
+  // 3 = Use only HTTP/3 QUIC
+  int use_http_version;
 
   // Print statistic interval
   int stats_interval;