diff --git a/client.go b/client.go
index 779558c..d7defc4 100644
--- a/client.go
+++ b/client.go
@@ -497,6 +497,7 @@ func NewClient(s *SFU, id string, name string, peerConnectionConfig webrtc.Confi
 				defer cancel()
 				<-ctx.Done()
 				client.stats.removeReceiverStats(remoteTrack.ID() + remoteTrack.RID())
+				client.tracks.remove([]string{remoteTrack.ID()})
 			}()
 
 			if err := client.tracks.Add(track); err != nil {
@@ -539,6 +540,8 @@ func NewClient(s *SFU, id string, name string, peerConnectionConfig webrtc.Confi
 					if simulcastTrack.remoteTrackLow != nil {
 						client.stats.removeReceiverStats(simulcastTrack.remoteTrackLow.track.ID() + simulcastTrack.remoteTrackLow.track.RID())
 					}
+
+					client.tracks.remove([]string{remoteTrack.ID()})
 				}()
 
 			} else if simulcast, ok = track.(*SimulcastTrack); ok {
@@ -1005,6 +1008,7 @@ func (c *Client) setClientTrack(t ITrack) iClientTrack {
 		defer func() {
 			c.muTracks.Lock()
 			delete(c.clientTracks, outputTrack.ID())
+			c.publishedTracks.remove([]string{outputTrack.ID()})
 			c.muTracks.Unlock()
 
 			c.renegotiate()
diff --git a/track.go b/track.go
index 9a95d6b..e68bf96 100644
--- a/track.go
+++ b/track.go
@@ -897,13 +897,6 @@ func (t *trackList) Add(track ITrack) error {
 
 	t.tracks[id] = track
 
-	go func() {
-		ctx, cancel := context.WithCancel(track.Context())
-		defer cancel()
-		<-ctx.Done()
-		t.remove([]string{id})
-	}()
-
 	return nil
 }