create topic and report topic

weed/mq/balancer/allocate.go

@@ -3,18 +3,50 @@ package balancer
 import (
 	cmap "github.com/orcaman/concurrent-map/v2"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"math/rand"
 )
 
-func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int) (assignments []*mq_pb.BrokerPartitionAssignment) {
-	return []*mq_pb.BrokerPartitionAssignment{
-		{
-			LeaderBroker:    "localhost:17777",
-			FollowerBrokers: []string{"localhost:17777"},
+func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int32) (assignments []*mq_pb.BrokerPartitionAssignment) {
+	// divide the ring into partitions
+	rangeSize := MaxPartitionCount / partitionCount
+	for i := int32(0); i < partitionCount; i++ {
+		assignment := &mq_pb.BrokerPartitionAssignment{
 			Partition: &mq_pb.Partition{
 				RingSize:   MaxPartitionCount,
-				RangeStart: 0,
-				RangeStop:  MaxPartitionCount,
+				RangeStart: int32(i * rangeSize),
+				RangeStop:  int32((i + 1) * rangeSize),
 			},
-		},
+		}
+		if i == partitionCount-1 {
+			assignment.Partition.RangeStop = MaxPartitionCount
+		}
+		assignments = append(assignments, assignment)
 	}
+
+	// pick the brokers
+	pickedBrokers := pickBrokers(brokers, partitionCount)
+
+	// assign the partitions to brokers
+	for i, assignment := range assignments {
+		assignment.LeaderBroker = pickedBrokers[i]
+	}
+	return
+}
+
+// for now: randomly pick brokers
+// TODO pick brokers based on the broker stats
+func pickBrokers(brokers cmap.ConcurrentMap[string, *BrokerStats], count int32) []string {
+	candidates := make([]string, 0, brokers.Count())
+	for brokerStatsItem := range brokers.IterBuffered() {
+		candidates = append(candidates, brokerStatsItem.Key)
+	}
+	pickedBrokers := make([]string, 0, count)
+	for i := int32(0); i < count; i++ {
+		p := rand.Int() % len(candidates)
+		if p < 0 {
+			p = -p
+		}
+		pickedBrokers = append(pickedBrokers, candidates[p])
+	}
+	return pickedBrokers
 }

weed/mq/balancer/allocate_test.go

@@ -24,12 +24,11 @@ func Test_allocateOneBroker(t *testing.T) {
 			name: "test only one broker",
 			args: args{
 				brokers:        brokers,
-				partitionCount: 6,
+				partitionCount: 1,
 			},
 			wantAssignments: []*mq_pb.BrokerPartitionAssignment{
 				{
-					LeaderBroker:    "localhost:17777",
-					FollowerBrokers: []string{"localhost:17777"},
+					LeaderBroker: "localhost:17777",
 					Partition: &mq_pb.Partition{
 						RingSize:   MaxPartitionCount,
 						RangeStart: 0,

weed/mq/balancer/balancer.go

@@ -7,7 +7,8 @@ import (
 )
 
 const (
-	MaxPartitionCount = 8 * 9 * 5 * 7 //2520
+	MaxPartitionCount  = 8 * 9 * 5 * 7 //2520
+	LockBrokerBalancer = "broker_balancer"
 )
 
 type Balancer struct {
@@ -51,6 +52,11 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
 
 }
 
+func (bs *BrokerStats) String() string {
+	return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}",
+		bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.Stats.Items())
+}
+
 type TopicPartition struct {
 	Namespace  string
 	Topic      string

weed/mq/balancer/lookup.go

@@ -1,10 +1,15 @@
 package balancer
 
 import (
+	"errors"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 )
 
-func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish bool) (assignments []*mq_pb.BrokerPartitionAssignment, err error) {
+var (
+	ErrNoBroker = errors.New("no broker")
+)
+
+func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish bool, partitionCount int32) (assignments []*mq_pb.BrokerPartitionAssignment, err error) {
 	// find existing topic partition assignments
 	for brokerStatsItem := range b.Brokers.IterBuffered() {
 		broker, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
@@ -39,5 +44,8 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
 	//   if the request is_for_subscribe
 	//     return error not found
 	// t := topic.FromPbTopic(request.Topic)
-	return allocateTopicPartitions(b.Brokers, 6), nil
+	if b.Brokers.IsEmpty() {
+		return nil, ErrNoBroker
+	}
+	return allocateTopicPartitions(b.Brokers, partitionCount), nil
 }

weed/mq/broker/broker_grpc_balancer.go

@@ -43,6 +43,7 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
 			brokerStats.UpdateStats(receivedStats)
 
 			glog.V(3).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
+			glog.V(3).Infof("received stats: %+v", receivedStats)
 		}
 	}
 

weed/mq/broker/broker_grpc_create.go

@@ -0,0 +1,56 @@
+package broker
+
+import (
+	"context"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// CreateTopic Runs on any broker, but proxied to the balancer if not the balancer
+func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_pb.CreateTopicRequest) (resp *mq_pb.CreateTopicResponse, err error) {
+	if broker.currentBalancer == "" {
+		return nil, status.Errorf(codes.Unavailable, "no balancer")
+	}
+	if !broker.lockAsBalancer.IsLocked() {
+		proxyErr := broker.withBrokerClient(false, broker.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+			resp, err = client.CreateTopic(ctx, request)
+			return nil
+		})
+		if proxyErr != nil {
+			return nil, proxyErr
+		}
+		return resp, err
+	}
+
+	ret := &mq_pb.CreateTopicResponse{}
+	ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
+
+	for _, bpa := range ret.BrokerPartitionAssignments {
+		if doCreateErr := broker.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
+			_, doCreateErr := client.DoCreateTopic(ctx, &mq_pb.DoCreateTopicRequest{
+				Topic:     request.Topic,
+				Partition: bpa.Partition,
+			})
+			return doCreateErr
+		}); doCreateErr != nil {
+			return nil, doCreateErr
+		}
+	}
+
+	return ret, err
+}
+
+func (broker *MessageQueueBroker) DoCreateTopic(ctx context.Context, req *mq_pb.DoCreateTopicRequest) (resp *mq_pb.DoCreateTopicResponse, err error) {
+	ret := &mq_pb.DoCreateTopicResponse{}
+	t, p := topic.FromPbTopic(req.Topic), topic.FromPbPartition(req.Partition)
+	localTopicPartition := broker.localTopicManager.GetTopicPartition(t, p)
+	if localTopicPartition == nil {
+		localTopicPartition = topic.NewLocalPartition(t, p, true, nil)
+		broker.localTopicManager.AddTopicPartition(t, localTopicPartition)
+	}
+
+	return ret, err
+}

weed/mq/broker/broker_grpc_lookup.go

@@ -36,7 +36,7 @@ func (broker *MessageQueueBroker) LookupTopicBrokers(ctx context.Context, reques
 
 	ret := &mq_pb.LookupTopicBrokersResponse{}
 	ret.Topic = request.Topic
-	ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(ret.Topic, request.IsForPublish)
+	ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(ret.Topic, request.IsForPublish, 6)
 	return ret, err
 }
 
@@ -45,3 +45,38 @@ func (broker *MessageQueueBroker) CheckTopicPartitionsStatus(c context.Context,
 	ret := &mq_pb.CheckTopicPartitionsStatusResponse{}
 	return ret, nil
 }
+
+func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.ListTopicsRequest) (resp *mq_pb.ListTopicsResponse, err error) {
+	if broker.currentBalancer == "" {
+		return nil, status.Errorf(codes.Unavailable, "no balancer")
+	}
+	if !broker.lockAsBalancer.IsLocked() {
+		proxyErr := broker.withBrokerClient(false, broker.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+			resp, err = client.ListTopics(ctx, request)
+			return nil
+		})
+		if proxyErr != nil {
+			return nil, proxyErr
+		}
+		return resp, err
+	}
+
+	ret := &mq_pb.ListTopicsResponse{}
+	knownTopics := make(map[*mq_pb.Topic]struct{})
+	for brokerStatsItem := range broker.Balancer.Brokers.IterBuffered() {
+		_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
+		for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
+			topicPartitionStat := topicPartitionStatsItem.Val
+			topic := &mq_pb.Topic{
+				Namespace: topicPartitionStat.TopicPartition.Namespace,
+				Name:      topicPartitionStat.TopicPartition.Topic,
+			}
+			if _, found := knownTopics[topic]; found {
+				continue
+			}
+			ret.Topics = append(ret.Topics, topic)
+		}
+	}
+
+	return ret, nil
+}

weed/mq/broker/broker_server.go

@@ -70,7 +70,7 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial
 		glog.V(1).Infof("broker %s found filer %s", self, mqBroker.currentFiler)
 
 		lockClient := cluster.NewLockClient(grpcDialOption, mqBroker.currentFiler)
-		mqBroker.lockAsBalancer = lockClient.StartLock(LockBrokerBalancer, self)
+		mqBroker.lockAsBalancer = lockClient.StartLock(balancer.LockBrokerBalancer, self)
 		for {
 			err := mqBroker.BrokerConnectToBalancer(self)
 			if err != nil {

weed/mq/broker/broker_stats.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/balancer"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -11,17 +12,13 @@ import (
 	"time"
 )
 
-const (
-	LockBrokerBalancer = "broker_balancer"
-)
-
 // BrokerConnectToBalancer connects to the broker balancer and sends stats
 func (broker *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
 	// find the lock owner
 	var brokerBalancer string
 	err := broker.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
-			Name: LockBrokerBalancer,
+			Name: balancer.LockBrokerBalancer,
 		})
 		if err != nil {
 			return err
@@ -32,6 +29,7 @@ func (broker *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
 	if err != nil {
 		return err
 	}
+	broker.currentBalancer = pb.ServerAddress(brokerBalancer)
 
 	glog.V(1).Infof("broker %s found balancer %s", self, brokerBalancer)
 

weed/pb/mq.proto

@@ -26,6 +26,12 @@ service SeaweedMessaging {
     // control plane for topic partitions
     rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
     }
+    rpc CreateTopic (CreateTopicRequest) returns (CreateTopicResponse) {
+    }
+    rpc DoCreateTopic (DoCreateTopicRequest) returns (DoCreateTopicResponse) {
+    }
+    rpc ListTopics (ListTopicsRequest) returns (ListTopicsResponse) {
+    }
     // a pub client will call this to get the topic partitions assignment
     rpc RequestTopicPartitions (RequestTopicPartitionsRequest) returns (RequestTopicPartitionsResponse) {
     }
@@ -128,6 +134,24 @@ message ConnectToBalancerRequest {
 message ConnectToBalancerResponse {
 }
 //////////////////////////////////////////////////
+message CreateTopicRequest {
+    Topic topic = 1;
+    int32 partition_count = 2;
+}
+message CreateTopicResponse {
+    repeated BrokerPartitionAssignment broker_partition_assignments = 2;
+}
+message DoCreateTopicRequest {
+    Topic topic = 1;
+    Partition partition = 2;
+}
+message DoCreateTopicResponse {
+}
+message ListTopicsRequest {
+}
+message ListTopicsResponse {
+    repeated Topic topics = 1;
+}
 message LookupTopicBrokersRequest {
     Topic topic = 1;
     bool is_for_publish = 2;

weed/pb/mq_pb/mq_grpc.pb.go

@@ -31,6 +31,9 @@ type SeaweedMessagingClient interface {
 	ConnectToBalancer(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_ConnectToBalancerClient, error)
 	// control plane for topic partitions
 	LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error)
+	CreateTopic(ctx context.Context, in *CreateTopicRequest, opts ...grpc.CallOption) (*CreateTopicResponse, error)
+	DoCreateTopic(ctx context.Context, in *DoCreateTopicRequest, opts ...grpc.CallOption) (*DoCreateTopicResponse, error)
+	ListTopics(ctx context.Context, in *ListTopicsRequest, opts ...grpc.CallOption) (*ListTopicsResponse, error)
 	// a pub client will call this to get the topic partitions assignment
 	RequestTopicPartitions(ctx context.Context, in *RequestTopicPartitionsRequest, opts ...grpc.CallOption) (*RequestTopicPartitionsResponse, error)
 	AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error)
@@ -124,6 +127,33 @@ func (c *seaweedMessagingClient) LookupTopicBrokers(ctx context.Context, in *Loo
 	return out, nil
 }
 
+func (c *seaweedMessagingClient) CreateTopic(ctx context.Context, in *CreateTopicRequest, opts ...grpc.CallOption) (*CreateTopicResponse, error) {
+	out := new(CreateTopicResponse)
+	err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/CreateTopic", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) DoCreateTopic(ctx context.Context, in *DoCreateTopicRequest, opts ...grpc.CallOption) (*DoCreateTopicResponse, error) {
+	out := new(DoCreateTopicResponse)
+	err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/DoCreateTopic", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) ListTopics(ctx context.Context, in *ListTopicsRequest, opts ...grpc.CallOption) (*ListTopicsResponse, error) {
+	out := new(ListTopicsResponse)
+	err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ListTopics", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedMessagingClient) RequestTopicPartitions(ctx context.Context, in *RequestTopicPartitionsRequest, opts ...grpc.CallOption) (*RequestTopicPartitionsResponse, error) {
 	out := new(RequestTopicPartitionsResponse)
 	err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/RequestTopicPartitions", in, out, opts...)
@@ -227,6 +257,9 @@ type SeaweedMessagingServer interface {
 	ConnectToBalancer(SeaweedMessaging_ConnectToBalancerServer) error
 	// control plane for topic partitions
 	LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error)
+	CreateTopic(context.Context, *CreateTopicRequest) (*CreateTopicResponse, error)
+	DoCreateTopic(context.Context, *DoCreateTopicRequest) (*DoCreateTopicResponse, error)
+	ListTopics(context.Context, *ListTopicsRequest) (*ListTopicsResponse, error)
 	// a pub client will call this to get the topic partitions assignment
 	RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error)
 	AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error)
@@ -259,6 +292,15 @@ func (UnimplementedSeaweedMessagingServer) ConnectToBalancer(SeaweedMessaging_Co
 func (UnimplementedSeaweedMessagingServer) LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method LookupTopicBrokers not implemented")
 }
+func (UnimplementedSeaweedMessagingServer) CreateTopic(context.Context, *CreateTopicRequest) (*CreateTopicResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateTopic not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) DoCreateTopic(context.Context, *DoCreateTopicRequest) (*DoCreateTopicResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DoCreateTopic not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) ListTopics(context.Context, *ListTopicsRequest) (*ListTopicsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListTopics not implemented")
+}
 func (UnimplementedSeaweedMessagingServer) RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method RequestTopicPartitions not implemented")
 }
@@ -403,6 +445,60 @@ func _SeaweedMessaging_LookupTopicBrokers_Handler(srv interface{}, ctx context.C
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedMessaging_CreateTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateTopicRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).CreateTopic(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/messaging_pb.SeaweedMessaging/CreateTopic",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).CreateTopic(ctx, req.(*CreateTopicRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_DoCreateTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DoCreateTopicRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).DoCreateTopic(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/messaging_pb.SeaweedMessaging/DoCreateTopic",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).DoCreateTopic(ctx, req.(*DoCreateTopicRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_ListTopics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListTopicsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).ListTopics(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/messaging_pb.SeaweedMessaging/ListTopics",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).ListTopics(ctx, req.(*ListTopicsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _SeaweedMessaging_RequestTopicPartitions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(RequestTopicPartitionsRequest)
 	if err := dec(in); err != nil {
@@ -531,6 +627,18 @@ var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "LookupTopicBrokers",
 			Handler:    _SeaweedMessaging_LookupTopicBrokers_Handler,
 		},
+		{
+			MethodName: "CreateTopic",
+			Handler:    _SeaweedMessaging_CreateTopic_Handler,
+		},
+		{
+			MethodName: "DoCreateTopic",
+			Handler:    _SeaweedMessaging_DoCreateTopic_Handler,
+		},
+		{
+			MethodName: "ListTopics",
+			Handler:    _SeaweedMessaging_ListTopics_Handler,
+		},
 		{
 			MethodName: "RequestTopicPartitions",
 			Handler:    _SeaweedMessaging_RequestTopicPartitions_Handler,

weed/shell/command_mq_topic_create.go

@@ -0,0 +1,67 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"io"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicCreate{})
+}
+
+type commandMqTopicCreate struct {
+}
+
+func (c *commandMqTopicCreate) Name() string {
+	return "mq.topic.create"
+}
+
+func (c *commandMqTopicCreate) Help() string {
+	return `create a topic with a given name
+
+	Example:
+		mq.topic.create -namespace <namespace> -topic <topic_name> -partition_count <partition_count>
+`
+}
+
+func (c *commandMqTopicCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+
+	// parse parameters
+	mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	namespace := mqCommand.String("namespace", "", "namespace name")
+	topicName := mqCommand.String("topic", "", "topic name")
+	partitionCount := mqCommand.Int("partitionCount", 6, "partition count")
+	if err := mqCommand.Parse(args); err != nil {
+		return err
+	}
+
+	// find the broker balancer
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
+
+	// create topic
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		resp, err := client.CreateTopic(context.Background(), &mq_pb.CreateTopicRequest{
+			Topic: &mq_pb.Topic{
+				Namespace: *namespace,
+				Name:      *topicName,
+			},
+			PartitionCount: int32(*partitionCount),
+		})
+		if err != nil {
+			return err
+		}
+		output, _ := json.MarshalIndent(resp, "", "  ")
+		fmt.Fprintf(writer, "response:\n%+v\n", string(output))
+		return nil
+	})
+
+}

weed/shell/command_mq_topic_list.go

@@ -1,7 +1,12 @@
 package shell
 
 import (
+	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/mq/balancer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"io"
 )
 
@@ -20,9 +25,40 @@ func (c *commandMqTopicList) Help() string {
 	return `print out all topics`
 }
 
-func (c *commandMqTopicList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+func (c *commandMqTopicList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
 
-	fmt.Fprintf(writer, "%s\n", commandEnv.option.Directory)
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
 
-	return nil
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		resp, err := client.ListTopics(context.Background(), &mq_pb.ListTopicsRequest{})
+		if err != nil {
+			return err
+		}
+		if len(resp.Topics) == 0 {
+			fmt.Fprintf(writer, "no topics found\n")
+			return nil
+		}
+		for _, topic := range resp.Topics {
+			fmt.Fprintf(writer, "  %+v\n", topic)
+		}
+		return nil
+	})
+}
+
+func findBrokerBalancer(commandEnv *CommandEnv) (brokerBalancer string, err error) {
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
+			Name: balancer.LockBrokerBalancer,
+		})
+		if err != nil {
+			return err
+		}
+		brokerBalancer = resp.Owner
+		return nil
+	})
+	return
 }