net: new downloader

The new downloader is replacing the download_client library and is
based on that.

Internal restructuring:

* Restructuring of socket functions and files.
* Parse HTTP header line for line. This reduces the size requirement
  for the download client recv buffer.
* Change TLS range override logic.
* Use range requests for nRF91 TLS only, and when specified by app.

API updates:

* Let application provide client buffer. This allows for multiple
  download clients with different buffer sizes.
* Add downloader_deinit()
* Add downloader_stop()
* Remove downloader_disconnect()
* Changed signature of downloader_init(), downloader_start()
  and downloader_get() to take a URI.
* Added downloader_get_with_host_and_path() for downloads where
  host and path are separate arguments to keep backwards compatibility.
* The transports (http, coap) are now separated out of the download
  client with its own API.

Future work:

* Take uri as input param to fota_download library and use URI in
  other relevant libaries and structures.
* Curent download client is deprecated and will be removed later.

Signed-off-by: Eivind Jølsgard <[email protected]>

Author: eivindj-nordic

.github/test-spec.yml

@@ -38,7 +38,7 @@
   - "lib/nrf_modem_lib/**/*"
   - "lib/sms/**/*"
   - "subsys/dfu/dfu_target/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
   - "subsys/net/lib/fota_download/**/*"
   - "subsys/net/lib/ftp_client/**/*"
   - "subsys/net/lib/nrf_cloud/**/*"
@@ -51,7 +51,7 @@
   - "lib/modem_info/**/*"
   - "lib/modem_key_mgmt/**/*"
   - "lib/date_time/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
   - "subsys/net/lib/fota_download/**/*"
   - "subsys/net/lib/lwm2m_client_utils/**/*"
   - "subsys/dfu/dfu_target/**/*"
@@ -81,7 +81,7 @@
   - "include/modem/nrf_modem_lib.h"
   - "include/modem/pdn.h"
   - "include/modem/sms.h"
-  - "include/net/download_client.h"
+  - "include/net/downloader.h"
   - "include/secure_services.h"
   - "lib/at_host/**/*"
   - "lib/at_monitor/**/*"
@@ -102,7 +102,7 @@
   - "samples/cellular/pdn/**/*"
   - "samples/cellular/sms/**/*"
   - "samples/cellular/udp/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
 
 "CI-lwm2m-test":
   - "include/modem/lte_lc*.h"
@@ -114,8 +114,8 @@
   - "samples/cellular/lwm2m_carrier/**/*"
   - "include/modem/at_monitor.h"
   - "lib/at_monitor/**/*"
-  - "include/net/download_client.h"
-  - "subsys/net/lib/download_client/**/*"
+  - "include/net/downloader.h"
+  - "subsys/net/lib/downloader/**/*"
   - "lib/sms/**/*"
   - "include/modem/sms.h"
   - "lib/pdn/**/*"
@@ -147,7 +147,7 @@
   - "subsys/mgmt/**/*"
   - "subsys/pcd/*"
   - "subsys/net/lib/*fota*/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
   - "tests/subsys/bootloader/**/*"
   - "tests/subsys/dfu/**/*"
 
@@ -433,7 +433,7 @@
   - "include/modem/nrf_modem_lib.h"
   - "include/modem/pdn.h"
   - "include/modem/sms.h"
-  - "include/net/download_client.h"
+  - "include/net/downloader.h"
   - "include/net/fota_download.h"
   - "include/net/nrf_cloud.h"
   - "include/net/nrf_cloud_agnss.h"
@@ -457,7 +457,7 @@
   - "lib/sms/**/*"
   - "lib/supl/**/*"
   - "samples/cellular/modem_shell/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
   - "subsys/net/lib/fota_download/**/*"
   - "subsys/net/lib/nrf_cloud/**/*"
   - "subsys/net/lib/rest_client/**/*"
@@ -509,7 +509,7 @@
   - "include/drivers/flash/**/*"
   - "include/flash_map_pm.h"
   - "include/modem/**/*"
-  - "include/net/download_client.h"
+  - "include/net/downloader.h"
   - "include/net/fota_download.h"
   - "include/net/nrf_cloud*"
   - "include/net/rest_client.h"
@@ -531,7 +531,7 @@
   - "subsys/bootloader/**/*"
   - "subsys/caf/**/*"
   - "subsys/dfu/**/*"
-  - "subsys/net/lib/download_client/**/*"
+  - "subsys/net/lib/downloader/**/*"
   - "subsys/net/lib/fota_download/**/*"
   - "subsys/net/lib/nrf_cloud/**/*"
   - "subsys/net/lib/nrf_provisioning/**/*"

CODEOWNERS

@@ -167,6 +167,7 @@
 /doc/nrf/libraries/networking/azure_*.rst @nrfconnect/ncs-cia-doc
 /doc/nrf/libraries/networking/coap_utils.rst @nrfconnect/ncs-terahertz-doc
 /doc/nrf/libraries/networking/download_client.rst @nrfconnect/ncs-modem-doc
+/doc/nrf/libraries/networking/downloader.rst @nrfconnect/ncs-modem-doc
 /doc/nrf/libraries/networking/fota_download.rst @nrfconnect/ncs-pluto-doc
 /doc/nrf/libraries/networking/ftp_client.rst @nrfconnect/ncs-iot-oulu-tampere-doc
 /doc/nrf/libraries/networking/icalendar_parser.rst @nrfconnect/ncs-doc-leads
@@ -335,7 +336,10 @@
 /include/mgmt/                            @nrfconnect/ncs-pluto
 /include/modem/                           @nrfconnect/ncs-modem
 /include/mpsl/                            @nrfconnect/ncs-dragoon
+/include/net/                             @nrfconnect/ncs-co-networking
 /include/net/azure_*                      @nrfconnect/ncs-cia
+/include/net/download_client*             @nrfconnect/ncs-modem
+/include/net/downloader*                  @nrfconnect/ncs-modem
 /include/net/nrf_cloud_*                  @nrfconnect/ncs-nrf-cloud
 /include/net/wifi_credentials.h           @nrfconnect/ncs-cia
 /include/nfc/                             @nrfconnect/ncs-co-drivers @nrfconnect/ncs-si-muffin
@@ -741,6 +745,8 @@
 /subsys/net/lib/mqtt_helper/              @nrfconnect/ncs-cia
 /subsys/net/lib/azure_*                   @nrfconnect/ncs-cia
 /subsys/net/lib/aws_*                     @nrfconnect/ncs-cia
+/subsys/net/lib/download_client*          @nrfconnect/ncs-modem
+/subsys/net/lib/downloader/               @nrfconnect/ncs-modem
 /subsys/net/lib/ftp_client/               @nrfconnect/ncs-iot-oulu
 /subsys/net/lib/hostap_crypto/            @krish2718 @jukkar @vivekuppunda
 /subsys/net/lib/icalendar_parser/         @lats1980
@@ -850,6 +856,8 @@
 /tests/subsys/mpsl/                       @nrfconnect/ncs-dragoon
 /tests/subsys/net/lib/aws_*/              @nrfconnect/ncs-cia
 /tests/subsys/net/lib/azure_iot_hub/      @nrfconnect/ncs-cia
+/tests/subsys/net/lib/downloader/         @nrfconnect/ncs-modem
+/tests/subsys/net/lib/download_client/    @nrfconnect/ncs-modem
 /tests/subsys/net/lib/fota_download/      @nrfconnect/ncs-pluto
 /tests/subsys/net/lib/lwm2m_*/            @nrfconnect/ncs-iot-oulu
 /tests/subsys/net/lib/mqtt_helper/        @nrfconnect/ncs-cia

applications/asset_tracker_v2/boards/native_sim.conf

@@ -103,8 +103,7 @@ CONFIG_NET_CONFIG_MY_IPV4_GW="192.0.2.2"
 # FOTA
 CONFIG_FOTA_DOWNLOAD=n
 CONFIG_DFU_TARGET=n
-CONFIG_DOWNLOAD_CLIENT=n
-CONFIG_DOWNLOAD_CLIENT_HTTP_FRAG_SIZE_1024=n
+CONFIG_DOWNLOADER=n
 
 # MCUBOOT
 CONFIG_BOOTLOADER_MCUBOOT=n

applications/asset_tracker_v2/doc/asset_tracker_v2_description.rst

@@ -235,7 +235,7 @@ This application uses the following |NCS| libraries and drivers:
 * :ref:`lib_date_time`
 * :ref:`lte_lc_readme`
 * :ref:`modem_info_readme`
-* :ref:`lib_download_client`
+* :ref:`lib_downloader`
 * :ref:`lib_fota_download`
 * :ref:`caf_leds`
 

applications/asset_tracker_v2/overlay-carrier.conf

@@ -18,8 +18,8 @@ CONFIG_PDN=y
 # AT Monitor is used by PDN library
 CONFIG_AT_MONITOR=y
 
-# Download client for DFU
-CONFIG_DOWNLOAD_CLIENT_MAX_FILENAME_SIZE=230
+# Downloader for DFU
+CONFIG_DOWNLOADER_MAX_FILENAME_SIZE=230
 
 # Modem info
 CONFIG_MODEM_INFO_BUFFER_SIZE=512
@@ -36,3 +36,6 @@ CONFIG_DFU_TARGET=y
 
 # Asserts
 CONFIG_ASSERT_VERBOSE=n
+
+# IPv4 (IPv6 is enabled by default)
+CONFIG_NET_IPV4=y

applications/asset_tracker_v2/prj.conf

@@ -60,11 +60,9 @@ CONFIG_FCB=y
 # FOTA
 CONFIG_FOTA_DOWNLOAD=y
 CONFIG_DFU_TARGET=y
-CONFIG_DOWNLOAD_CLIENT=y
-CONFIG_DOWNLOAD_CLIENT_HTTP_FRAG_SIZE_1024=y
-CONFIG_DOWNLOAD_CLIENT_STACK_SIZE=4096
-CONFIG_DOWNLOAD_CLIENT_BUF_SIZE=2300
-CONFIG_DOWNLOAD_CLIENT_MAX_HOSTNAME_SIZE=128
+CONFIG_DOWNLOADER=y
+CONFIG_DOWNLOADER_STACK_SIZE=4096
+CONFIG_DOWNLOADER_MAX_HOSTNAME_SIZE=128
 
 # Flash - Used in FOTA, settings and storage for P-GPS.
 CONFIG_FLASH=y

applications/serial_lte_modem/doc/slm_description.rst

@@ -684,7 +684,7 @@ This application uses the following |NCS| libraries:
 * :ref:`lib_ftp_client`
 * :ref:`sms_readme`
 * :ref:`lib_fota_download`
-* :ref:`lib_download_client`
+* :ref:`lib_downloader`
 * :ref:`lib_nrf_cloud`
 * :ref:`lib_nrf_cloud_agnss`
 * :ref:`lib_nrf_cloud_pgps`

applications/serial_lte_modem/overlay-carrier.conf

@@ -13,7 +13,7 @@ CONFIG_FP_HARDABI=y
 CONFIG_PDN=y
 
 # Download client for DFU
-CONFIG_DOWNLOAD_CLIENT_MAX_FILENAME_SIZE=230
+CONFIG_DOWNLOADER_MAX_FILENAME_SIZE=230
 
 # Modem info
 CONFIG_MODEM_INFO_BUFFER_SIZE=512
@@ -28,3 +28,6 @@ CONFIG_LWM2M_CARRIER_SETTINGS=y
 
 # DFU target library
 CONFIG_DFU_TARGET=y
+
+# IPv4 (IPv6 is enabled by default)
+CONFIG_NET_IPV4=y

applications/serial_lte_modem/prj.conf

@@ -62,9 +62,9 @@ CONFIG_HTTP_PARSER_URL=y
 CONFIG_FOTA_DOWNLOAD=y
 CONFIG_FOTA_DOWNLOAD_PROGRESS_EVT=y
 CONFIG_DFU_TARGET=y
-CONFIG_DOWNLOAD_CLIENT=y
-CONFIG_DOWNLOAD_CLIENT_STACK_SIZE=4096
-CONFIG_DOWNLOAD_CLIENT_MAX_FILENAME_SIZE=2048
+CONFIG_DOWNLOADER=y
+CONFIG_DOWNLOADER_STACK_SIZE=4096
+CONFIG_DOWNLOADER_MAX_FILENAME_SIZE=2048
 CONFIG_BOOTLOADER_MCUBOOT=y
 CONFIG_IMG_MANAGER=y
 

applications/serial_lte_modem/src/slm_at_fota.c

@@ -26,10 +26,10 @@
 LOG_MODULE_REGISTER(slm_fota, CONFIG_SLM_LOG_LEVEL);
 
 /* file_uri: scheme://hostname[:port]path[?parameters] */
-#define FILE_URI_MAX	CONFIG_DOWNLOAD_CLIENT_MAX_FILENAME_SIZE
+#define FILE_URI_MAX	CONFIG_DOWNLOADER_MAX_FILENAME_SIZE
 #define SCHEMA_HTTP     "http"
 #define SCHEMA_HTTPS	"https"
-#define URI_HOST_MAX	CONFIG_DOWNLOAD_CLIENT_MAX_HOSTNAME_SIZE
+#define URI_HOST_MAX	CONFIG_DOWNLOADER_MAX_HOSTNAME_SIZE
 #define URI_SCHEMA_MAX	8
 #define ERASE_POLL_TIME 2