Merge pull request #112 from TheThingsNetwork/chirpstack-v4

Support ChirpStack v4

Author: KrishnaIyer

CHANGELOG.md

@@ -8,14 +8,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Added
 
+- Support for ChirpStack v4.
+
 ### Changed
 
 ### Deprecated
 
 ### Removed
 
+- Support for ChirpStack v3. Use versions `0.11.x` for ChirpStack v3.
+
+### Fixed
+
+## [v0.11.2] (2024-03-04)
+
 ### Fixed
 
+- Exporting devices from ChirpStack v3.
+
 ## [v0.11.1] (2024-01-20)
 
 ### Fixed

README.md

@@ -11,7 +11,8 @@ Binaries are available on [GitHub](https://github.com/TheThingsNetwork/lorawan-s
 ## Support
 
 - [x] The Things Network Stack V2
-- [x] [ChirpStack Network Server](https://www.chirpstack.io/)
+- [x] [ChirpStack Network Server v4](https://www.chirpstack.io/)
+- [x] [ChirpStack Network Server v3](https://www.chirpstack.io/docs/v3-documentation.html) (only versions `v0.11.x`).
 - [x] [The Things Stack](https://www.github.com/TheThingsNetwork/lorawan-stack/)
 - [x] [Firefly](https://fireflyiot.com/)
 - [ ] [LORIOT Network Server](https://www.loriot.io/)
@@ -106,7 +107,9 @@ $ ttn-lw-migrate ttnv2 application 'my-app-id' --dry-run --verbose > devices.jso
 $ ttn-lw-migrate ttnv2 application 'my-app-id' > devices.json
 ```
 
-## ChirpStack
+## ChirpStack v3
+
+> Note: ChirpStack v3 support is removed from versions `v0.12.0` onwards. Use `v0.11.x` for ChirpStack v3.
 
 ### Configuration
 
@@ -176,6 +179,81 @@ And export with:
 $ ttn-lw-migrate chirpstack application < application_names.txt > devices.json
 ```
 
+## ChirpStack v4
+
+> Minimum supported version: `v0.12.0`
+
+### Configuration
+
+Configure with environment variables, or command-line arguments. See `--help` for more details:
+
+```bash
+$ export CHIRPSTACK_API_URL="localhost:8080"    # ChirpStack Application Server URL
+$ export CHIRPSTACK_API_KEY="eyJ0eX........"    # Generate from ChirpStack GUI
+$ export JOIN_EUI="0101010102020203"            # JoinEUI for exported devices
+$ export FREQUENCY_PLAN_ID="EU_863_870"         # Frequency Plan for exported devices
+$ export CHIRPSTACK_EXPORT_SESSION="true"       # Set to true for session migration.
+```
+
+See [Frequency Plans](https://thethingsstack.io/reference/frequency-plans/) for the list of frequency plans available on The Things Stack. For example, to use `United States 902-928 MHz, FSB 1`, you need to specify the `US_902_928_FSB_1` frequency plan ID.
+
+> _NOTE_: `JoinEUI` and `FrequencyPlanID` are required because ChirpStack does not store these fields.
+
+### Notes
+
+- ABP devices without an active session are successfully exported from ChirpStack, but cannot be imported into The Things Stack.
+- MaxEIRP may not be always set properly.
+- ChirpStack payload formatters also accept a `variables` parameter. This will always be `null` on The Things Stack.
+- ChirpStack v4 uses UUIDs as application ID. The migration tool uses the appends the last index of the UUID to application ID.
+  - Ex: If the ChirpStack v4 application ID is `59459ffa-bfd3-4ef3-9cee-e1ca219397f2`, the tool generates `chirpstack-e1ca219397f2` as the application ID.
+
+### Export Devices
+
+To export a single device using its DevEUI (e.g. `0102030405060708`):
+
+```
+$ ttn-lw-migrate chirpstack device '0102030405060708' > devices.json
+```
+
+In order to export a large number of devices, create a file named `device_euis.txt` with one DevEUI per line:
+
+```
+0102030405060701
+0102030405060702
+0102030405060703
+0102030405060704
+0102030405060705
+0102030405060706
+```
+
+And then export with:
+
+```bash
+$ ttn-lw-migrate chirpstack device < device_euis.txt > devices.json
+```
+
+### Export Applications
+
+Similarly, to export all devices of application `chirpstack-app-1`:
+
+```bash
+$ ttn-lw-migrate chirpstack application 'chirpstack-app-1' > devices.json
+```
+
+In order to export multiple applications, create a file named `application_names.txt` with one Application name per line:
+
+```
+chirpstack-app-1
+chirpstack-app-2
+chirpstack-app-3
+```
+
+And export with:
+
+```bash
+$ ttn-lw-migrate chirpstack application < application_names.txt > devices.json
+```
+
 ## The Things Stack
 
 ### Configuration

cmd/chirpstack/chirpstack.go

@@ -23,7 +23,7 @@ const sourceName = "chirpstack"
 
 // ChirpStackCmd represents the chirpstack source.
 var ChirpStackCmd = commands.Source(sourceName,
-	"Export devices from ChirpStack V3",
+	"Export devices from ChirpStack v4",
 	commands.WithDevicesOptions(
 		commands.WithShort("Export devices by DevEUI"),
 	),

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/TheThingsNetwork/ttn/core/types v0.0.0-20190516112328-fcd38e2b9dc6
 	github.com/apex/log v1.9.0
 	github.com/brocaar/chirpstack-api/go/v3 v3.12.5
+	github.com/chirpstack/chirpstack/api/go/v4 v4.6.0
 	github.com/mdempsky/unconvert v0.0.0-20230125054757-2661c2c99a9b
 	github.com/mgechev/revive v1.3.7
 	github.com/smarty/assertions v1.15.1

go.sum

@@ -177,6 +177,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc=
 github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww=
+github.com/chirpstack/chirpstack/api/go/v4 v4.6.0 h1:l+nr/QhFab1y9E8LVOJq/lDG+o0+mShcZOCNBvFYXUA=
+github.com/chirpstack/chirpstack/api/go/v4 v4.6.0/go.mod h1:6+68s1PGHq2QWZ216RTwXhp7h1vCiMc6kX3f4s74ZzQ=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

pkg/source/chirpstack/chirpstack.go

@@ -20,12 +20,12 @@ import (
 )
 
 func init() {
-	cfg, flags := config.New()
+	cfg := config.New()
 
 	source.RegisterSource(source.Registration{
 		Name:        "chirpstack",
-		Description: "Migrate from ChirpStack LoRaWAN Network Server",
-		FlagSet:     flags,
+		Description: "Migrate from ChirpStack LoRaWAN Network Server v4",
+		FlagSet:     cfg.Flags(),
 		Create:      createNewSource(cfg),
 	})
 }

pkg/source/chirpstack/config/config.go

@@ -24,6 +24,8 @@ import (
 
 	"github.com/spf13/pflag"
 	"go.thethings.network/lorawan-stack-migrate/pkg/source"
+	"go.thethings.network/lorawan-stack/v3/pkg/fetch"
+	"go.thethings.network/lorawan-stack/v3/pkg/frequencyplans"
 	"go.thethings.network/lorawan-stack/v3/pkg/types"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
@@ -32,96 +34,113 @@ import (
 
 const dialTimeout = 10 * time.Second
 
-func New() (*Config, *pflag.FlagSet) {
-	var (
-		config = &Config{}
-		flags  = &pflag.FlagSet{}
-	)
+type Config struct {
+	src source.Config
+
+	apiKey, caCertPath, url, joinEUI string
+	flags                            *pflag.FlagSet
+	FPStore                          *frequencyplans.Store
+	insecure                         bool
+
+	ClientConn *grpc.ClientConn
 
-	flags.StringVar(&config.url,
+	ExportVars,
+	ExportSession bool
+	FrequencyPlanID string
+	JoinEUI         *types.EUI64
+}
+
+func New() *Config {
+	config := &Config{
+		flags: &pflag.FlagSet{},
+	}
+
+	config.flags.StringVar(&config.url,
 		"api-url",
-		os.Getenv("CHIRPSTACK_API_URL"),
+		"",
 		"ChirpStack API URL")
-	flags.StringVar(&config.token,
-		"api-token",
-		os.Getenv("CHIRPSTACK_API_TOKEN"),
-		"ChirpStack API Token")
-	flags.StringVar(&config.caPath,
-		"api-ca",
-		os.Getenv("CHIRPSTACK_API_CA"),
-		"(optional) CA for TLS")
-	flags.BoolVar(&config.insecure,
-		"api-insecure",
-		os.Getenv("CHIRPSTACK_API_INSECURE") == "1",
+	config.flags.StringVar(&config.apiKey,
+		"api-key",
+		"",
+		"ChirpStack API key")
+	config.flags.StringVar(&config.caCertPath,
+		"ca-cert-path",
+		"",
+		"(optional) Path to the CA certificate file for ChirpStack API TLS connections")
+	config.flags.BoolVar(&config.insecure,
+		"insecure",
+		false,
 		"Do not connect to ChirpStack over TLS")
-	flags.BoolVar(&config.ExportVars,
+	config.flags.BoolVar(&config.ExportVars,
 		"export-vars",
 		false,
 		"Export device variables from ChirpStack")
-	flags.BoolVar(&config.ExportSession,
+	config.flags.BoolVar(&config.ExportSession,
 		"export-session",
-		true,
+		false,
 		"Export device session keys from ChirpStack")
-	flags.StringVar(&config.joinEUI,
+	config.flags.StringVar(&config.joinEUI,
 		"join-eui",
-		os.Getenv("JOIN_EUI"),
+		"",
 		"JoinEUI of exported devices")
-	flags.StringVar(&config.FrequencyPlanID,
+	config.flags.StringVar(&config.FrequencyPlanID,
 		"frequency-plan-id",
-		os.Getenv("FREQUENCY_PLAN_ID"),
+		"",
 		"Frequency Plan ID of exported devices")
 
-	return config, flags
+	return config
 }
 
-type Config struct {
-	source.Config
-
-	ClientConn *grpc.ClientConn
+func (c *Config) Initialize(src source.Config) error {
+	c.src = src
 
-	token, caPath, url,
-	FrequencyPlanID string
-
-	joinEUI string
-	JoinEUI *types.EUI64
-
-	insecure,
-	ExportVars,
-	ExportSession bool
-}
-
-func (c *Config) Initialize() error {
-	if c.token == "" {
+	if c.apiKey = os.Getenv("CHIRPSTACK_API_KEY"); c.apiKey == "" {
 		return errNoAPIToken.New()
 	}
-	if c.url == "" {
+	if c.url = os.Getenv("CHIRPSTACK_API_URL"); c.url == "" {
 		return errNoAPIURL.New()
 	}
-	if c.FrequencyPlanID == "" {
+	if c.FrequencyPlanID = os.Getenv("FREQUENCY_PLAN_ID"); c.FrequencyPlanID == "" {
 		return errNoFrequencyPlan.New()
 	}
-
+	if c.joinEUI = os.Getenv("JOIN_EUI"); c.joinEUI == "" {
+		return errNoJoinEUI.New()
+	}
 	c.JoinEUI = &types.EUI64{}
 	if err := c.JoinEUI.UnmarshalText([]byte(c.joinEUI)); err != nil {
 		return errInvalidJoinEUI.WithAttributes("join_eui", c.joinEUI)
 	}
+	c.caCertPath = os.Getenv("CHIRPSTACK_CA_CERT_PATH")
+	c.insecure = os.Getenv("CHIRPSTACK_INSECURE") == "true"
+	c.ExportSession = os.Getenv("EXPORT_SESSION") == "true"
+	c.ExportVars = os.Getenv("EXPORT_VARS") == "true"
+
 	err := c.dialGRPC(
 		grpc.FailOnNonTempDialError(true),
 		grpc.WithBlock(),
-		grpc.WithPerRPCCredentials(token(c.token)),
+		grpc.WithPerRPCCredentials(token(c.apiKey)),
 	)
 	if err != nil {
 		return err
 	}
-
+	fpFetcher, err := fetch.FromHTTP(http.DefaultClient, src.FrequencyPlansURL)
+	if err != nil {
+		return err
+	}
+	c.FPStore = frequencyplans.NewStore(fpFetcher)
 	return nil
 }
 
+// Flags returns the flags for the configuration.
+func (c *Config) Flags() *pflag.FlagSet {
+	return c.flags
+}
+
 func (c *Config) dialGRPC(opts ...grpc.DialOption) error {
 	if c.insecure {
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	} else {
-		tlsConfig, err := generateTLSConfig(c.caPath)
+		tlsConfig, err := generateTLSConfig(c.caCertPath)
 		if err != nil {
 			return err
 		}

pkg/source/chirpstack/config/errors.go

@@ -20,6 +20,7 @@ var (
 	errNoAPIToken      = errors.DefineInvalidArgument("no_api_token", "no API token")
 	errNoAPIURL        = errors.DefineInvalidArgument("no_api_url", "no API URL")
 	errNoFrequencyPlan = errors.DefineInvalidArgument("no_frequency_plan", "no Frequency Plan")
+	errNoJoinEUI       = errors.DefineInvalidArgument("no_join_eui", "no join eui")
 
 	errInvalidJoinEUI = errors.DefineInvalidArgument("invalid_join_eui", "invalid JoinEUI `{join_eui}`")
 )

pkg/source/chirpstack/errors.go

@@ -25,11 +25,12 @@ var (
 
 	errAPI = errors.Define("api", "API error")
 
-	errAppNotFound       = errors.DefineNotFound("app_not_found", "app `{app}` not found")
-	errInvalidDevAddr    = errors.DefineInvalidArgument("invalid_dev_addr", "invalid DevAddr `{dev_addr}`")
-	errInvalidDevEUI     = errors.DefineInvalidArgument("invalid_dev_eui", "invalid DevEUI `{dev_eui}`")
-	errInvalidJoinEUI    = errors.DefineInvalidArgument("invalid_join_eui", "invalid JoinEUI `{join_eui}`")
-	errInvalidPHYVersion = errors.DefineInvalidArgument("invalid_phy_version", "invalid PHY version `{phy_version}`")
-	errInvalidMACVersion = errors.DefineInvalidArgument("invalid_mac_version", "invalid MAC version `{mac_version}`")
-	errInvalidKey        = errors.DefineInvalidArgument("invalid_key", "invalid key `{key}`")
+	errAppNotFound          = errors.DefineNotFound("app_not_found", "app `{app}` not found")
+	errInvalidDevAddr       = errors.DefineInvalidArgument("invalid_dev_addr", "invalid DevAddr `{dev_addr}`")
+	errInvalidDevEUI        = errors.DefineInvalidArgument("invalid_dev_eui", "invalid DevEUI `{dev_eui}`")
+	errInvalidJoinEUI       = errors.DefineInvalidArgument("invalid_join_eui", "invalid JoinEUI `{join_eui}`")
+	errInvalidPHYVersion    = errors.DefineInvalidArgument("invalid_phy_version", "invalid PHY version `{phy_version}`")
+	errInvalidMACVersion    = errors.DefineInvalidArgument("invalid_mac_version", "invalid MAC version `{mac_version}`")
+	errInvalidKey           = errors.DefineInvalidArgument("invalid_key", "invalid key `{key}`")
+	errInvalidApplicationID = errors.DefineInvalidArgument("invalid_application_id", "invalid application ID `{application_id}`")
 )