Merge pull request TheThingsNetwork#2233 from TheThingsIndustries/mer…

…ge/open-source-master

CHANGELOG.md

@@ -9,6 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [3.8.5] - 2020-07-06
+
+### Added
+
 - Option to reset end device payload formatters in the Console.
 - Sentry reporting to device claiming frontend.
 - Service discovery using DNS SRV records for external Application Server linking.
@@ -31,8 +45,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Defer events subscriptions until there is actual interest for events.
 - End device creation form with wizard in the Console.
 
-### Deprecated
-
 ### Removed
 
 - Requirement to specify `frequency_plan_id` when creating gateways in the Console.
@@ -921,7 +933,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 NOTE: These links should respect backports. See https://github.com/TheThingsNetwork/lorawan-stack/pull/1444/files#r333379706.
 -->
 
-[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.8.4...HEAD
+[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.8.5...HEAD
+[3.8.5]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.8.4...v3.8.5
 [3.8.4]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.8.3...v3.8.4
 [3.8.3]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.8.2...v3.8.3
 [3.8.2]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.7.2...v3.8.2

config/messages.json

@@ -4607,6 +4607,15 @@
       "file": "hooks.tti.go"
     }
   },
+  "error:pkg/identityserver/store:eui_taken": {
+    "translations": {
+      "en": "EUI already taken"
+    },
+    "description": {
+      "package": "pkg/identityserver/store",
+      "file": "store.go"
+    }
+  },
   "error:pkg/identityserver/store:gateway_eui_not_found": {
     "translations": {
       "en": "gateway EUI `{eui}` not found"
@@ -4796,6 +4805,24 @@
       "file": "identityserver.go"
     }
   },
+  "error:pkg/identityserver:end_device_euis_taken": {
+    "translations": {
+      "en": "an end device with JoinEUI `{join_eui}` and DevEUI `{dev_eui}` is already registered as `{device_id}` in application `{application_id}`"
+    },
+    "description": {
+      "package": "pkg/identityserver",
+      "file": "end_device_registry.go"
+    }
+  },
+  "error:pkg/identityserver:gateway_eui_taken": {
+    "translations": {
+      "en": "a gateway with EUI `{gateway_eui}` is already registered as `{gateway_id}`"
+    },
+    "description": {
+      "package": "pkg/identityserver",
+      "file": "gateway_registry.go"
+    }
+  },
   "error:pkg/identityserver:insufficient_tenant_rights": {
     "translations": {
       "en": "insufficient tenant rights"

doc/config/_default/config.toml

@@ -27,7 +27,7 @@ pygmentsUseClasses = true
   keywords = []
   github_repository = "https://github.com/TheThingsNetwork/lorawan-stack"
   github_repository_edit = "https://github.com/TheThingsNetwork/lorawan-stack/edit/master/doc/content"
-  version = "v3.8.4"
+  version = "v3.8.5"
 
 [markup]
   [markup.goldmark]

doc/content/integrations/mqtt/_index.md

@@ -3,13 +3,15 @@ title: "MQTT Server"
 description: ""
 ---
 
-The Application Server exposes a MQTT server to work with streaming events. In order to use the MQTT server you need to create a new API key to authenticate.
+{{% tts %}} exposes an MQTT server to work with streaming events. This section explains how to connect an MQTT client and subscribe to uplinks or publish downlinks.
+
+>{{% tts %}} supports the [MQTT Standard Version 3.1.1](http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.pdf).
 
 <!--more-->
 
 ## Creating an API Key
 
-The Console provides the required connection information and can be used to create an API key for authentication. In your application select the **MQTT** submenu from the **Integrations** side menu.
+ In order to use the MQTT server you need to create a new API key to authenticate. The Console provides the required connection information and can be used to create an API key for authentication. In your application select the **MQTT** submenu from the **Integrations** side menu.
 
 {{< figure src="mqtt-integration.png" alt="MQTT connection information" >}}
 

doc/content/integrations/webhooks/_index.md

@@ -4,47 +4,3 @@ description: ""
 ---
 
 The webhooks feature allows the Application Server to send application related messages to specific HTTP(S) endpoints.
-
-<!--more-->
-
-## Creating a Webhook
-
-Creating a webhook requires you to have an HTTP(S) endpoint available.
-
-In your application select the **Webhooks** submenu from the **Integrations** side menu. Clicking on the **+ Add Webhook** button will open the Webhook creation screen. Fill in your webhook ID, format and base URL.
-
-{{< figure src="webhook-creation.png" alt="Webhook creation screen" >}}
-
-The paths are appended to the base URL. So, the Application Server will perform `POST` requests on the endpoint `https://app.example.com/lorahooks/join` for join-accepts and `https://app.example.com/lorahooks/up` for uplink messages. Clicking the **Add Webhook** button will create the Webhook.
-
->Note: If you don't have an endpoint available for testing, use for example [PostBin](https://postb.in).
-
-## Scheduling Downlinks
-
-You can schedule downlink messages using webhooks too. This requires an API key with traffic writing rights, which can be created using the Console. In your application, select the **API Keys** sidemenu and click on the **+ Add API Key** button. You can now fill in the name and the rights of your API key.
-
-{{< figure src="api-key-creation.png" alt="API key creation screen" >}}
-
-Click on the **Create API Key** button in order to create the API key. This will open the API key information screen.
-
-{{< figure src="api-key-created.png" alt="API key created" >}}
-
-Make sure to save your API key at this point, since it will no longer be retrievable after you leave the page. You can now pass the API key as bearer token on the `Authorization` header.
-
-The downlink queue operation paths are:
-
-- For push: `/api/v3/as/applications/{application_id}/webhooks/{webhook_id}/devices/{device_id}/down/push`
-- For replace: `/api/v3/as/applications/{application_id}/webhooks/{webhook_id}/devices/{device_id}/down/replace`
-
-For example:
-
-```
-$ curl https://thethings.example.com/api/v3/as/applications/app1/webhooks/wh1/devices/dev1/down/push \
-  -X POST \
-  -H 'Authorization: Bearer NNSXS.VEEBURF3KR77ZR..' \
-  --data '{"downlinks":[{"frm_payload":"vu8=","f_port":15,"priority":"NORMAL"}]}'
-```
-
-Will push a downlink to the end device `dev1` of the application `app1` using the webhook `wh1`.
-
-You can also save the API key in the webhook configuration page using the the **Downlink API Key** field. The Application Server will provide it to your endpoint using the `X-Downlink-Apikey` header and the push and replace operations paths using the `X-Downlink-Push` and `X-Downlink-Replace` headers.

doc/content/integrations/webhooks/creating-webhooks.md

@@ -0,0 +1,19 @@
+---
+title: "Creating Webhooks"
+description: ""
+weight: -1
+---
+
+This section provides instructions for creating a webhook in the console.
+
+<!--more-->
+
+Creating a webhook requires you to have an HTTP(S) endpoint available.
+
+In your application select the **Webhooks** submenu from the **Integrations** side menu. Clicking on the **+ Add Webhook** button will open the Webhook creation screen. Fill in your webhook ID, format and base URL.
+
+{{< figure src="../webhook-creation.png" alt="Webhook creation screen" >}}
+
+The paths are appended to the base URL. So, the Application Server will perform `POST` requests on the endpoint `https://app.example.com/lorahooks/join` for join-accepts and `https://app.example.com/lorahooks/up` for uplink messages. Clicking the **Add Webhook** button will create the Webhook.
+
+>Note: If you don't have an endpoint available for testing, you can test with a free service like [PostBin](https://postb.in).

doc/content/integrations/webhooks/path-variables.md

@@ -0,0 +1,24 @@
+---
+title: "Webhook Path Variables"
+description: ""
+weight: -1
+---
+
+Webhook path variables allow you to substitute device and application specific variables in webhook paths. This section provides instructions for using webhook path variables.
+
+<!--more-->
+
+Webhook path variables allow you to use the following variables in webhook paths:
+
+- `appID`
+- `appEUI`
+- `joinEUI`
+- `devID`
+- `devEUI`
+- `devAddr`
+
+Path variables can be inserted in the **Base URL** webhook field or the **Path** field for a particular type of message.
+
+For example, if the **Base URL** is `https://app.example.com/lorahooks{/appID}` and the **Path** is `/up{/devID}` an uplink from the device `dev1` of application `app1` will be posted at `https://app.example.com/lorahooks/app1/up/dev1`.
+
+See [IETF RFC65700](https://tools.ietf.org/html/rfc6570) for more documentation about URL path variables. {{% tts %}} supports all forms of path variable substitution.

doc/content/integrations/webhooks/scheduling-downlinks.md

@@ -0,0 +1,37 @@
+---
+title: "Scheduling Downlinks"
+description: ""
+weight: -1
+---
+
+This section provides instructions for creating scheduling downlinks using webhooks.
+
+<!--more-->
+
+You can schedule downlink messages using webhooks. This requires an API key with traffic writing rights, which can be created using the Console. In your application, select the **API Keys** sidemenu and click on the **+ Add API Key** button. You can now fill in the name and the rights of your API key.
+
+{{< figure src="../api-key-creation.png" alt="API key creation screen" >}}
+
+Click on the **Create API Key** button in order to create the API key. This will open the API key information screen.
+
+{{< figure src="../api-key-created.png" alt="API key created" >}}
+
+Make sure to save your API key at this point, since it will no longer be retrievable after you leave the page. You can now pass the API key as bearer token on the `Authorization` header.
+
+The downlink queue operation paths are:
+
+- For push: `/api/v3/as/applications/{application_id}/webhooks/{webhook_id}/devices/{device_id}/down/push`
+- For replace: `/api/v3/as/applications/{application_id}/webhooks/{webhook_id}/devices/{device_id}/down/replace`
+
+For example:
+
+```
+$ curl https://thethings.example.com/api/v3/as/applications/app1/webhooks/wh1/devices/dev1/down/push \
+  -X POST \
+  -H 'Authorization: Bearer NNSXS.VEEBURF3KR77ZR..' \
+  --data '{"downlinks":[{"frm_payload":"vu8=","f_port":15,"priority":"NORMAL"}]}'
+```
+
+Will push a downlink to the end device `dev1` of the application `app1` using the webhook `wh1`.
+
+You can also save the API key in the webhook configuration page using the the **Downlink API Key** field. The Application Server will provide it to your endpoint using the `X-Downlink-Apikey` header and the push and replace operations paths using the `X-Downlink-Push` and `X-Downlink-Replace` headers.

doc/content/integrations/webhooks/webhook-templates/_index.md

@@ -5,18 +5,12 @@ summary: Webhook templates define a webhook integration that is not created (yet
 weight: 1
 ---
 
-This is the reference for Webhook Templates
-
-It covers the format of the templates and how the template instantiation process works.
-
-## What is it?
-
 Webhook templates define a webhook integration that is not created (yet). Templates allows for using common values for many webhooks, such as a common base URLs.
 
-## Who is it for?
-
 Webhook templates are primarily targeted at service providers who want to create specialized webhook integrations for the users of {{% tts %}}.
 
+<!--more-->
+
 ### Typical use cases
 
 1. Create a webhook with a personalized base URL, format and message paths.

doc/themes/the-things-stack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hugo-theme-the-things-stack",
-  "version": "3.8.4",
+  "version": "3.8.5",
   "private": true,
   "description": "Hugo Theme for The Things Stack",
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ttn-stack",
-  "version": "3.8.4",
+  "version": "3.8.5",
   "description": "The Things Stack",
   "main": "index.js",
   "repository": "https://github.com/TheThingsNetwork/lorawan-stack.git",

pkg/identityserver/end_device_registry.go

@@ -21,6 +21,7 @@ import (
 	"github.com/gogo/protobuf/types"
 	"github.com/jinzhu/gorm"
 	"go.thethings.network/lorawan-stack/v3/pkg/auth/rights"
+	"go.thethings.network/lorawan-stack/v3/pkg/errors"
 	"go.thethings.network/lorawan-stack/v3/pkg/events"
 	"go.thethings.network/lorawan-stack/v3/pkg/identityserver/blacklist"
 	"go.thethings.network/lorawan-stack/v3/pkg/identityserver/store"
@@ -42,6 +43,11 @@ var (
 	)
 )
 
+var errEndDeviceEUIsTaken = errors.DefineAlreadyExists(
+	"end_device_euis_taken",
+	"an end device with JoinEUI `{join_eui}` and DevEUI `{dev_eui}` is already registered as `{device_id}` in application `{application_id}`",
+)
+
 func (is *IdentityServer) createEndDevice(ctx context.Context, req *ttnpb.CreateEndDeviceRequest) (dev *ttnpb.EndDevice, err error) {
 	if err = rights.RequireApplication(ctx, req.EndDeviceIdentifiers.ApplicationIdentifiers, ttnpb.RIGHT_APPLICATION_DEVICES_WRITE); err != nil {
 		return nil, err
@@ -65,6 +71,19 @@ func (is *IdentityServer) createEndDevice(ctx context.Context, req *ttnpb.Create
 		return nil
 	})
 	if err != nil {
+		if errors.IsAlreadyExists(err) && errors.Resemble(err, store.ErrEUITaken) {
+			if ids, err := is.getEndDeviceIdentifiersForEUIs(ctx, &ttnpb.GetEndDeviceIdentifiersForEUIsRequest{
+				JoinEUI: *req.JoinEUI,
+				DevEUI:  *req.DevEUI,
+			}); err == nil {
+				return nil, errEndDeviceEUIsTaken.WithAttributes(
+					"join_eui", req.JoinEUI.String(),
+					"dev_eui", req.DevEUI.String(),
+					"device_id", ids.GetDeviceID(),
+					"application_id", ids.GetApplicationID(),
+				)
+			}
+		}
 		return nil, err
 	}
 	events.Publish(evtCreateEndDevice(ctx, req.EndDeviceIdentifiers, nil))

pkg/identityserver/end_device_registry_test.go

@@ -20,11 +20,11 @@ import (
 
 	pbtypes "github.com/gogo/protobuf/types"
 	"github.com/smartystreets/assertions"
-	"github.com/smartystreets/assertions/should"
 	"go.thethings.network/lorawan-stack/v3/pkg/errors"
 	"go.thethings.network/lorawan-stack/v3/pkg/ttnpb"
 	"go.thethings.network/lorawan-stack/v3/pkg/types"
 	"go.thethings.network/lorawan-stack/v3/pkg/util/test"
+	"go.thethings.network/lorawan-stack/v3/pkg/util/test/assertions/should"
 	"google.golang.org/grpc"
 )
 
@@ -174,6 +174,22 @@ func TestEndDevicesCRUD(t *testing.T) {
 			a.So(*ids, should.Resemble, created.EndDeviceIdentifiers)
 		}
 
+		_, err = reg.Create(ctx, &ttnpb.CreateEndDeviceRequest{
+			EndDevice: ttnpb.EndDevice{
+				EndDeviceIdentifiers: ttnpb.EndDeviceIdentifiers{
+					DeviceID:               "other-test-device-id",
+					ApplicationIdentifiers: app.ApplicationIdentifiers,
+					JoinEUI:                &joinEUI,
+					DevEUI:                 &devEUI,
+				},
+				Name: "test-device-name",
+			},
+		}, creds)
+
+		if a.So(err, should.NotBeNil) {
+			a.So(err, should.HaveSameErrorDefinitionAs, errEndDeviceEUIsTaken)
+		}
+
 		list, err := reg.List(ctx, &ttnpb.ListEndDevicesRequest{
 			FieldMask:              pbtypes.FieldMask{Paths: []string{"name"}},
 			ApplicationIdentifiers: app.ApplicationIdentifiers,

pkg/identityserver/gateway_registry.go

@@ -20,6 +20,7 @@ import (
 	"github.com/gogo/protobuf/types"
 	"github.com/jinzhu/gorm"
 	"go.thethings.network/lorawan-stack/v3/pkg/auth/rights"
+	"go.thethings.network/lorawan-stack/v3/pkg/errors"
 	"go.thethings.network/lorawan-stack/v3/pkg/events"
 	"go.thethings.network/lorawan-stack/v3/pkg/identityserver/blacklist"
 	"go.thethings.network/lorawan-stack/v3/pkg/identityserver/store"
@@ -41,6 +42,8 @@ var (
 	)
 )
 
+var errGatewayEUITaken = errors.DefineAlreadyExists("gateway_eui_taken", "a gateway with EUI `{gateway_eui}` is already registered as `{gateway_id}`")
+
 func (is *IdentityServer) createGateway(ctx context.Context, req *ttnpb.CreateGatewayRequest) (gtw *ttnpb.Gateway, err error) {
 	if err = blacklist.Check(ctx, req.GatewayID); err != nil {
 		return nil, err
@@ -84,6 +87,16 @@ func (is *IdentityServer) createGateway(ctx context.Context, req *ttnpb.CreateGa
 		return nil
 	})
 	if err != nil {
+		if errors.IsAlreadyExists(err) && errors.Resemble(err, store.ErrEUITaken) {
+			if ids, err := is.getGatewayIdentifiersForEUI(ctx, &ttnpb.GetGatewayIdentifiersForEUIRequest{
+				EUI: *req.EUI,
+			}); err == nil {
+				return nil, errGatewayEUITaken.WithAttributes(
+					"gateway_eui", req.EUI.String(),
+					"gateway_id", ids.GetGatewayID(),
+				)
+			}
+		}
 		return nil, err
 	}
 	events.Publish(evtCreateGateway(ctx, req.GatewayIdentifiers, nil))