clone

server/internal/topics/trie.go

@@ -0,0 +1,344 @@
+package topics
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/mochi-co/mqtt/server/internal/packets"
+)
+
+// Subscriptions is a map of subscriptions keyed on client.
+type Subscriptions map[string]byte
+
+// Index is a prefix/trie tree containing topic subscribers and retained messages.
+type Index struct {
+	mu   sync.RWMutex // a mutex for locking the whole index.
+	Root *Leaf        // a leaf containing a message and more leaves.
+}
+
+// New returns a pointer to a new instance of Index.
+func New() *Index {
+	return &Index{
+		Root: &Leaf{
+			Leaves:  make(map[string]*Leaf),
+			Clients: make(map[string]byte),
+		},
+	}
+}
+
+// RetainMessage saves a message payload to the end of a topic branch. Returns
+// 1 if a retained message was added, and -1 if the retained message was removed.
+// 0 is returned if sequential empty payloads are received.
+func (x *Index) RetainMessage(msg packets.Packet) int64 {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	n := x.poperate(msg.TopicName)
+
+	// If there is a payload, we can store it.
+	if len(msg.Payload) > 0 {
+		n.Message = msg
+		return 1
+	}
+
+	// Otherwise, we are unsetting it.
+	// If there was a previous retained message, return -1 instead of 0.
+	var r int64 = 0
+	if len(n.Message.Payload) > 0 && n.Message.FixedHeader.Retain == true {
+		r = -1
+	}
+	x.unpoperate(msg.TopicName, "", true)
+
+	return r
+}
+
+// Subscribe creates a subscription filter for a client. Returns true if the
+// subscription was new.
+func (x *Index) Subscribe(filter, client string, qos byte) bool {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+
+	n := x.poperate(filter)
+	_, ok := n.Clients[client]
+	n.Clients[client] = qos
+	n.Filter = filter
+
+	return !ok
+}
+
+// Unsubscribe removes a subscription filter for a client. Returns true if an
+// unsubscribe action successful and the subscription existed.
+func (x *Index) Unsubscribe(filter, client string) bool {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+
+	n := x.poperate(filter)
+	_, ok := n.Clients[client]
+
+	return x.unpoperate(filter, client, false) && ok
+}
+
+// unpoperate steps backward through a trie sequence and removes any orphaned
+// nodes. If a client id is specified, it will unsubscribe a client. If message
+// is true, it will delete a retained message.
+func (x *Index) unpoperate(filter string, client string, message bool) bool {
+	var d int // Walk to end leaf.
+	var particle string
+	var hasNext = true
+	e := x.Root
+	for hasNext {
+		particle, hasNext = isolateParticle(filter, d)
+		d++
+		e, _ = e.Leaves[particle]
+
+		// If the topic part doesn't exist in the tree, there's nothing
+		// left to do.
+		if e == nil {
+			return false
+		}
+	}
+
+	// Step backward removing client and orphaned leaves.
+	var key string
+	var orphaned bool
+	var end = true
+	for e.Parent != nil {
+		key = e.Key
+
+		// Wipe the client from this leaf if it's the filter end.
+		if end {
+			if client != "" {
+				delete(e.Clients, client)
+			}
+			if message {
+				e.Message = packets.Packet{}
+			}
+			end = false
+		}
+
+		// If this leaf is empty, note it as orphaned.
+		orphaned = len(e.Clients) == 0 && len(e.Leaves) == 0 && !e.Message.FixedHeader.Retain
+
+		// Traverse up the branch.
+		e = e.Parent
+
+		// If the leaf we just came from was empty, delete it.
+		if orphaned {
+			delete(e.Leaves, key)
+		}
+	}
+
+	return true
+
+}
+
+// poperate iterates and populates through a topic/filter path, instantiating
+// leaves as it goes and returning the final leaf in the branch.
+// poperate is a more enjoyable word than iterpop.
+func (x *Index) poperate(topic string) *Leaf {
+	var d int
+	var particle string
+	var hasNext = true
+	n := x.Root
+	for hasNext {
+		particle, hasNext = isolateParticle(topic, d)
+		d++
+
+		child, _ := n.Leaves[particle]
+		if child == nil {
+			child = &Leaf{
+				Key:     particle,
+				Parent:  n,
+				Leaves:  make(map[string]*Leaf),
+				Clients: make(map[string]byte),
+			}
+			n.Leaves[particle] = child
+		}
+		n = child
+	}
+
+	return n
+}
+
+// Subscribers returns a map of clients who are subscribed to matching filters.
+func (x *Index) Subscribers(topic string) Subscriptions {
+	x.mu.RLock()
+	defer x.mu.RUnlock()
+	return x.Root.scanSubscribers(topic, 0, make(Subscriptions))
+}
+
+// Messages returns a slice of retained topic messages which match a filter.
+func (x *Index) Messages(filter string) []packets.Packet {
+	// ReLeaf("messages", x.Root, 0)
+	x.mu.RLock()
+	defer x.mu.RUnlock()
+	return x.Root.scanMessages(filter, 0, make([]packets.Packet, 0, 32))
+}
+
+// Leaf is a child node on the tree.
+type Leaf struct {
+	Message packets.Packet   // a message which has been retained for a specific topic.
+	Key     string           // the key that was used to create the leaf.
+	Filter  string           // the path of the topic filter being matched.
+	Parent  *Leaf            // a pointer to the parent node for the leaf.
+	Leaves  map[string]*Leaf // a map of child nodes, keyed on particle id.
+	Clients map[string]byte  // a map of client ids subscribed to the topic.
+}
+
+// scanSubscribers recursively steps through a branch of leaves finding clients who
+// have subscription filters matching a topic, and their highest QoS byte.
+func (l *Leaf) scanSubscribers(topic string, d int, clients Subscriptions) Subscriptions {
+	part, hasNext := isolateParticle(topic, d)
+
+	// For either the topic part, a +, or a #, follow the branch.
+	for _, particle := range []string{part, "+", "#"} {
+
+		// Topics beginning with the reserved $ character are restricted from
+		// being returned for top level wildcards.
+		if d == 0 && len(part) > 0 && part[0] == '$' && (particle == "+" || particle == "#") {
+			continue
+		}
+
+		if child, ok := l.Leaves[particle]; ok {
+
+			// We're only interested in getting clients from the final
+			// element in the topic, or those with wildhashes.
+			if !hasNext || particle == "#" {
+
+				// Capture the highest QOS byte for any client with a filter
+				// matching the topic.
+				for client, qos := range child.Clients {
+					if ex, ok := clients[client]; !ok || ex < qos {
+						clients[client] = qos
+					}
+				}
+
+				// Make sure we also capture any client who are listening
+				// to this topic via path/#
+				if !hasNext {
+					if extra, ok := child.Leaves["#"]; ok {
+						for client, qos := range extra.Clients {
+							if ex, ok := clients[client]; !ok || ex < qos {
+								clients[client] = qos
+							}
+						}
+					}
+				}
+			}
+
+			// If this branch has hit a wildhash, just return immediately.
+			if particle == "#" {
+				return clients
+			} else if hasNext {
+				clients = child.scanSubscribers(topic, d+1, clients)
+			}
+		}
+	}
+
+	return clients
+}
+
+// scanMessages recursively steps through a branch of leaves finding retained messages
+// that match a topic filter. Setting `d` to -1 will enable wildhash mode, and will
+// recursively check ALL child leaves in every subsequent branch.
+func (l *Leaf) scanMessages(filter string, d int, messages []packets.Packet) []packets.Packet {
+
+	// If a wildhash mode has been set, continue recursively checking through all
+	// child leaves regardless of their particle key.
+	if d == -1 {
+		for _, child := range l.Leaves {
+			if child.Message.FixedHeader.Retain {
+				messages = append(messages, child.Message)
+			}
+			messages = child.scanMessages(filter, -1, messages)
+		}
+		return messages
+	}
+
+	// Otherwise, we'll get the particle for d in the filter.
+	particle, hasNext := isolateParticle(filter, d)
+
+	// If there's no more particles after this one, then take the messages from
+	// these topics.
+	if !hasNext {
+
+		// Wildcards and Wildhashes must be checked first, otherwise they
+		// may be detected as standard particles, and not act properly.
+		if particle == "+" || particle == "#" {
+
+			// Otherwise, if it's a wildcard or wildhash, get messages from all
+			// the child leaves. This wildhash captures messages on the actual
+			// wildhash position, whereas the d == -1 block collects subsequent
+			// messages further down the branch.
+			for _, child := range l.Leaves {
+				if d == 0 && len(child.Key) > 0 && child.Key[0] == '$' {
+					continue
+				}
+				if child.Message.FixedHeader.Retain {
+					messages = append(messages, child.Message)
+				}
+			}
+		} else if child, ok := l.Leaves[particle]; ok {
+			if child.Message.FixedHeader.Retain {
+				messages = append(messages, child.Message)
+			}
+		}
+
+	} else {
+
+		// If it's not the last particle, branch out to the next leaves, scanning
+		// all available if it's a wildcard, or just one if it's a specific particle.
+		if particle == "+" {
+			for _, child := range l.Leaves {
+				if d == 0 && len(child.Key) > 0 && child.Key[0] == '$' {
+					continue
+				}
+				messages = child.scanMessages(filter, d+1, messages)
+			}
+		} else if child, ok := l.Leaves[particle]; ok {
+			messages = child.scanMessages(filter, d+1, messages)
+		}
+	}
+
+	// If the particle was a wildhash, scan all the child leaves setting the
+	// d value to wildhash mode.
+	if particle == "#" {
+		for _, child := range l.Leaves {
+			if d == 0 && len(child.Key) > 0 && child.Key[0] == '$' {
+				continue
+			}
+			messages = child.scanMessages(filter, -1, messages)
+		}
+	}
+
+	return messages
+}
+
+// isolateParticle extracts a particle between d / and d+1 / without allocations.
+func isolateParticle(filter string, d int) (particle string, hasNext bool) {
+	var next, end int
+	for i := 0; end > -1 && i <= d; i++ {
+		end = strings.IndexRune(filter, '/')
+		if d > -1 && i == d && end > -1 {
+			hasNext = true
+			particle = filter[next:end]
+		} else if end > -1 {
+			hasNext = false
+			filter = filter[end+1:]
+		} else {
+			hasNext = false
+			particle = filter[next:]
+		}
+	}
+
+	return
+}
+
+// ReLeaf is a dev function for showing the trie leafs.
+/*
+func ReLeaf(m string, leaf *Leaf, d int) {
+	for k, v := range leaf.Leaves {
+		fmt.Println(m, d, strings.Repeat("  ", d), k)
+		ReLeaf(m, v, d+1)
+	}
+}
+*/

server/internal/topics/trie_test.go

@@ -0,0 +1,494 @@
+package topics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mochi-co/mqtt/server/internal/packets"
+)
+
+func TestNew(t *testing.T) {
+	index := New()
+	require.NotNil(t, index)
+	require.NotNil(t, index.Root)
+}
+
+func BenchmarkNew(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		New()
+	}
+}
+
+func TestPoperate(t *testing.T) {
+	index := New()
+	child := index.poperate("path/to/my/mqtt")
+	require.Equal(t, "mqtt", child.Key)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"])
+
+	child = index.poperate("a/b/c/d/e")
+	require.Equal(t, "e", child.Key)
+	child = index.poperate("a/b/c/c/a")
+	require.Equal(t, "a", child.Key)
+}
+
+func BenchmarkPoperate(b *testing.B) {
+	index := New()
+	for n := 0; n < b.N; n++ {
+		index.poperate("path/to/my/mqtt")
+	}
+}
+
+func TestUnpoperate(t *testing.T) {
+	index := New()
+	index.Subscribe("path/to/my/mqtt", "client-1", 0)
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Clients, "client-1")
+
+	index.Subscribe("path/to/another/mqtt", "client-1", 0)
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Clients, "client-1")
+
+	pk := packets.Packet{TopicName: "path/to/retained/message", Payload: []byte{'h', 'e', 'l', 'l', 'o'}}
+	index.RetainMessage(pk)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["retained"].Leaves["message"])
+	require.Equal(t, pk, index.Root.Leaves["path"].Leaves["to"].Leaves["retained"].Leaves["message"].Message)
+
+	pk2 := packets.Packet{TopicName: "path/to/my/mqtt", Payload: []byte{'s', 'h', 'a', 'r', 'e', 'd'}}
+	index.RetainMessage(pk2)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"])
+	require.Equal(t, pk2, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Message)
+
+	index.unpoperate("path/to/my/mqtt", "", true) // delete retained
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Clients, "client-1")
+	require.Equal(t, false, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Message.FixedHeader.Retain)
+
+	index.unpoperate("path/to/my/mqtt", "client-1", false) // unsubscribe client
+	require.Nil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"])
+
+	index.unpoperate("path/to/retained/message", "", true) // delete retained
+	require.NotContains(t, index.Root.Leaves["path"].Leaves["to"].Leaves, "my")
+
+	index.unpoperate("path/to/whatever", "client-1", false) // unsubscribe client
+	require.Nil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"])
+
+	//require.Empty(t, index.Root.Leaves["path"])
+
+}
+
+func BenchmarkUnpoperate(b *testing.B) {
+	index := New()
+	for n := 0; n < b.N; n++ {
+		index.poperate("path/to/my/mqtt")
+	}
+}
+
+func TestRetainMessage(t *testing.T) {
+	pk := packets.Packet{
+		FixedHeader: packets.FixedHeader{
+			Retain: true,
+		},
+		TopicName: "path/to/my/mqtt",
+		Payload:   []byte{'h', 'e', 'l', 'l', 'o'},
+	}
+	pk2 := packets.Packet{
+		FixedHeader: packets.FixedHeader{
+			Retain: true,
+		},
+		TopicName: "path/to/another/mqtt",
+		Payload:   []byte{'h', 'e', 'l', 'l', 'o'},
+	}
+
+	index := New()
+	q := index.RetainMessage(pk)
+	require.Equal(t, int64(1), q)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"])
+	require.Equal(t, pk, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Message)
+
+	index.Subscribe("path/to/another/mqtt", "client-1", 0)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Clients["client-1"])
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"])
+
+	q = index.RetainMessage(pk2)
+	require.Equal(t, int64(1), q)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"])
+	require.Equal(t, pk2, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Message)
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Clients, "client-1")
+
+	// The same message already exists, but we're not doing a deep-copy check, so it's considered
+	// to be a new message.
+	q = index.RetainMessage(pk2)
+	require.Equal(t, int64(1), q)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"])
+	require.Equal(t, pk2, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Message)
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Clients, "client-1")
+
+	// Delete retained
+	pk3 := packets.Packet{TopicName: "path/to/another/mqtt", Payload: []byte{}}
+	q = index.RetainMessage(pk3)
+	require.Equal(t, int64(-1), q)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"])
+	require.Equal(t, pk, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Message)
+	require.Equal(t, false, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Message.FixedHeader.Retain)
+
+	// Second Delete retained
+	q = index.RetainMessage(pk3)
+	require.Equal(t, int64(0), q)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"])
+	require.Equal(t, pk, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Message)
+	require.Equal(t, false, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Message.FixedHeader.Retain)
+
+}
+
+func BenchmarkRetainMessage(b *testing.B) {
+	index := New()
+	pk := packets.Packet{TopicName: "path/to/another/mqtt"}
+	for n := 0; n < b.N; n++ {
+		index.RetainMessage(pk)
+	}
+}
+
+func TestSubscribeOK(t *testing.T) {
+	index := New()
+
+	q := index.Subscribe("path/to/my/mqtt", "client-1", 0)
+	require.Equal(t, true, q)
+
+	q = index.Subscribe("path/to/my/mqtt", "client-1", 0)
+	require.Equal(t, false, q)
+
+	q = index.Subscribe("path/to/my/mqtt", "client-2", 0)
+	require.Equal(t, true, q)
+
+	q = index.Subscribe("path/to/another/mqtt", "client-1", 0)
+	require.Equal(t, true, q)
+
+	q = index.Subscribe("path/+", "client-2", 0)
+	require.Equal(t, true, q)
+
+	q = index.Subscribe("#", "client-3", 0)
+	require.Equal(t, true, q)
+
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Clients, "client-1")
+	require.Equal(t, "path/to/my/mqtt", index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Filter)
+	require.Equal(t, "mqtt", index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Key)
+	require.Equal(t, index.Root.Leaves["path"], index.Root.Leaves["path"].Leaves["to"].Parent)
+	require.NotNil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Clients, "client-2")
+
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["another"].Leaves["mqtt"].Clients, "client-1")
+	require.Contains(t, index.Root.Leaves["path"].Leaves["+"].Clients, "client-2")
+	require.Contains(t, index.Root.Leaves["#"].Clients, "client-3")
+}
+
+func BenchmarkSubscribe(b *testing.B) {
+	index := New()
+	for n := 0; n < b.N; n++ {
+		index.Subscribe("path/to/mqtt/basic", "client-1", 0)
+	}
+}
+
+func TestUnsubscribeA(t *testing.T) {
+	index := New()
+	index.Subscribe("path/to/my/mqtt", "client-1", 0)
+	index.Subscribe("path/to/+/mqtt", "client-1", 0)
+	index.Subscribe("path/to/stuff", "client-1", 0)
+	index.Subscribe("path/to/stuff", "client-2", 0)
+	index.Subscribe("#", "client-3", 0)
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"].Leaves["mqtt"].Clients, "client-1")
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["+"].Leaves["mqtt"].Clients, "client-1")
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["stuff"].Clients, "client-1")
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["stuff"].Clients, "client-2")
+	require.Contains(t, index.Root.Leaves["#"].Clients, "client-3")
+
+	ok := index.Unsubscribe("path/to/my/mqtt", "client-1")
+	require.Equal(t, true, ok)
+
+	require.Nil(t, index.Root.Leaves["path"].Leaves["to"].Leaves["my"])
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["+"].Leaves["mqtt"].Clients, "client-1")
+
+	ok = index.Unsubscribe("path/to/stuff", "client-1")
+	require.Equal(t, true, ok)
+
+	require.NotContains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["stuff"].Clients, "client-1")
+	require.Contains(t, index.Root.Leaves["path"].Leaves["to"].Leaves["stuff"].Clients, "client-2")
+	require.Contains(t, index.Root.Leaves["#"].Clients, "client-3")
+
+	ok = index.Unsubscribe("fdasfdas/dfsfads/sa", "client-1")
+	require.Equal(t, false, ok)
+
+}
+
+func TestUnsubscribeCascade(t *testing.T) {
+	index := New()
+	index.Subscribe("a/b/c", "client-1", 0)
+	index.Subscribe("a/b/c/e/e", "client-1", 0)
+
+	ok := index.Unsubscribe("a/b/c/e/e", "client-1")
+	require.Equal(t, true, ok)
+	require.NotEmpty(t, index.Root.Leaves)
+	require.Contains(t, index.Root.Leaves["a"].Leaves["b"].Leaves["c"].Clients, "client-1")
+}
+
+// This benchmark is Unsubscribe-Subscribe
+func BenchmarkUnsubscribe(b *testing.B) {
+	index := New()
+
+	for n := 0; n < b.N; n++ {
+		index.Subscribe("path/to/my/mqtt", "client-1", 0)
+		index.Unsubscribe("path/to/mqtt/basic", "client-1")
+	}
+}
+
+func TestSubscribersFind(t *testing.T) {
+	tt := []struct {
+		filter string
+		topic  string
+		len    int
+	}{
+		{
+			filter: "a",
+			topic:  "a",
+			len:    1,
+		},
+		{
+			filter: "a/",
+			topic:  "a",
+			len:    0,
+		},
+		{
+			filter: "a/",
+			topic:  "a/",
+			len:    1,
+		},
+		{
+			filter: "/a",
+			topic:  "/a",
+			len:    1,
+		},
+		{
+			filter: "path/to/my/mqtt",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "path/to/+/mqtt",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "+/to/+/mqtt",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "#",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "+/+/+/+",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "+/+/+/#",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "zen/#",
+			topic:  "zen",
+			len:    1,
+		},
+		{
+			filter: "+/+/#",
+			topic:  "path/to/my/mqtt",
+			len:    1,
+		},
+		{
+			filter: "path/to/",
+			topic:  "path/to/my/mqtt",
+			len:    0,
+		},
+		{
+			filter: "#/stuff",
+			topic:  "path/to/my/mqtt",
+			len:    0,
+		},
+		{
+			filter: "$SYS/#",
+			topic:  "$SYS/info",
+			len:    1,
+		},
+		{
+			filter: "#",
+			topic:  "$SYS/info",
+			len:    0,
+		},
+		{
+			filter: "+/info",
+			topic:  "$SYS/info",
+			len:    0,
+		},
+	}
+
+	for i, check := range tt {
+		index := New()
+		index.Subscribe(check.filter, "client-1", 0)
+		clients := index.Subscribers(check.topic)
+		//spew.Dump(clients)
+		require.Equal(t, check.len, len(clients), "Unexpected clients len at %d %s %s", i, check.filter, check.topic)
+	}
+
+}
+
+func BenchmarkSubscribers(b *testing.B) {
+	index := New()
+	index.Subscribe("path/to/my/mqtt", "client-1", 0)
+	index.Subscribe("path/to/+/mqtt", "client-1", 0)
+	index.Subscribe("something/things/stuff/+", "client-1", 0)
+	index.Subscribe("path/to/stuff", "client-2", 0)
+	index.Subscribe("#", "client-3", 0)
+
+	for n := 0; n < b.N; n++ {
+		index.Subscribers("path/to/testing/mqtt")
+	}
+}
+
+func TestIsolateParticle(t *testing.T) {
+	particle, hasNext := isolateParticle("path/to/my/mqtt", 0)
+	require.Equal(t, "path", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("path/to/my/mqtt", 1)
+	require.Equal(t, "to", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("path/to/my/mqtt", 2)
+	require.Equal(t, "my", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("path/to/my/mqtt", 3)
+	require.Equal(t, "mqtt", particle)
+	require.Equal(t, false, hasNext)
+
+	particle, hasNext = isolateParticle("/path/", 0)
+	require.Equal(t, "", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("/path/", 1)
+	require.Equal(t, "path", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("/path/", 2)
+	require.Equal(t, "", particle)
+	require.Equal(t, false, hasNext)
+
+	particle, hasNext = isolateParticle("a/b/c/+/+", 3)
+	require.Equal(t, "+", particle)
+	require.Equal(t, true, hasNext)
+	particle, hasNext = isolateParticle("a/b/c/+/+", 4)
+	require.Equal(t, "+", particle)
+	require.Equal(t, false, hasNext)
+}
+
+func BenchmarkIsolateParticle(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		isolateParticle("path/to/my/mqtt", 3)
+	}
+}
+
+func TestMessagesPattern(t *testing.T) {
+	tt := []struct {
+		packet packets.Packet
+		filter string
+		len    int
+	}{
+		{
+			packets.Packet{TopicName: "a/b/c/d", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"a/b/c/d",
+			1,
+		},
+		{
+			packets.Packet{TopicName: "a/b/c/e", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"a/+/c/+",
+			2,
+		},
+		{
+			packets.Packet{TopicName: "a/b/d/f", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"+/+/+/+",
+			3,
+		},
+		{
+			packets.Packet{TopicName: "q/w/e/r/t/y", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"q/w/e/#",
+			1,
+		},
+		{
+			packets.Packet{TopicName: "q/w/x/r/t/x", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"q/#",
+			2,
+		},
+		{
+			packets.Packet{TopicName: "asd", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"asd",
+			1,
+		},
+		{
+			packets.Packet{TopicName: "$SYS/testing", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"#",
+			8,
+		},
+		{
+			packets.Packet{TopicName: "$SYS/test", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"+/testing",
+			0,
+		},
+		{
+			packets.Packet{TopicName: "$SYS/info", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"$SYS/info",
+			1,
+		},
+		{
+			packets.Packet{TopicName: "$SYS/b", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"$SYS/#",
+			4,
+		},
+		{
+			packets.Packet{TopicName: "asd/fgh/jkl", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"#",
+			8,
+		},
+		{
+			packets.Packet{TopicName: "stuff/asdadsa/dsfdsafdsadfsa/dsfdsf/sdsadas", Payload: []byte{'h', 'e', 'l', 'l', 'o'}, FixedHeader: packets.FixedHeader{Retain: true}},
+			"stuff/#/things", // indexer will ignore trailing /things
+			1,
+		},
+	}
+	index := New()
+	for _, check := range tt {
+		index.RetainMessage(check.packet)
+	}
+
+	for i, check := range tt {
+		messages := index.Messages(check.filter)
+		require.Equal(t, check.len, len(messages), "Unexpected messages len at %d %s %s", i, check.filter, check.packet.TopicName)
+	}
+}
+
+func TestMessagesFind(t *testing.T) {
+	index := New()
+	index.RetainMessage(packets.Packet{TopicName: "a/a", Payload: []byte{'a'}, FixedHeader: packets.FixedHeader{Retain: true}})
+	index.RetainMessage(packets.Packet{TopicName: "a/b", Payload: []byte{'b'}, FixedHeader: packets.FixedHeader{Retain: true}})
+	messages := index.Messages("a/a")
+	require.Equal(t, 1, len(messages))
+
+	messages = index.Messages("a/+")
+	require.Equal(t, 2, len(messages))
+}
+
+func BenchmarkMessages(b *testing.B) {
+	index := New()
+	index.RetainMessage(packets.Packet{TopicName: "path/to/my/mqtt"})
+	index.RetainMessage(packets.Packet{TopicName: "path/to/another/mqtt"})
+	index.RetainMessage(packets.Packet{TopicName: "path/a/some/mqtt"})
+	index.RetainMessage(packets.Packet{TopicName: "what/is"})
+	index.RetainMessage(packets.Packet{TopicName: "q/w/e/r/t/y"})
+
+	for n := 0; n < b.N; n++ {
+		index.Messages("path/to/+/mqtt")
+	}
+}