diff --git a/server_config.go b/server_config.go
index ee808fe9..02ded9d6 100644
--- a/server_config.go
+++ b/server_config.go
@@ -57,11 +57,12 @@ func (c *PacketConnConfig) validate() error {
 	if c.PacketConn == nil {
 		return errConnUnset
 	}
-	if c.RelayAddressGenerator == nil {
-		return errRelayAddressGeneratorUnset
+
+	if c.RelayAddressGenerator != nil {
+		return c.RelayAddressGenerator.Validate()
 	}
 
-	return c.RelayAddressGenerator.Validate()
+	return nil
 }
 
 // ListenerConfig is a single net.Listener to accept connections on. This will be used for TCP, TLS and DTLS listeners