1
0
mirror of https://github.com/mainflux/mainflux.git synced 2025-04-24 13:48:49 +08:00

159 lines
3.7 KiB
Go
Raw Normal View History

// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package consumer
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
import (
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
"encoding/json"
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
"fmt"
"github.com/go-redis/redis"
"github.com/mainflux/mainflux/bootstrap"
"github.com/mainflux/mainflux/logger"
)
const (
stream = "mainflux.things"
group = "mainflux.bootstrap"
thingPrefix = "thing."
thingRemove = thingPrefix + "remove"
thingDisconnect = thingPrefix + "disconnect"
channelPrefix = "channel."
channelUpdate = channelPrefix + "update"
channelRemove = channelPrefix + "remove"
exists = "BUSYGROUP Consumer Group name already exists"
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
)
// EventStore represents event source for things and channels provisioning.
type EventStore interface {
// Subscribes to given subject and receives events.
Subscribe(string) error
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
}
type eventStore struct {
svc bootstrap.Service
client *redis.Client
consumer string
logger logger.Logger
}
// NewEventStore returns new event store instance.
func NewEventStore(svc bootstrap.Service, client *redis.Client, consumer string, log logger.Logger) EventStore {
return eventStore{
svc: svc,
client: client,
consumer: consumer,
logger: log,
}
}
func (es eventStore) Subscribe(subject string) error {
err := es.client.XGroupCreateMkStream(stream, group, "$").Err()
if err != nil && err.Error() != exists {
return err
}
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
for {
streams, err := es.client.XReadGroup(&redis.XReadGroupArgs{
Group: group,
Consumer: es.consumer,
Streams: []string{stream, ">"},
Count: 100,
}).Result()
if err != nil || len(streams) == 0 {
continue
}
for _, msg := range streams[0].Messages {
event := msg.Values
var err error
switch event["operation"] {
case thingRemove:
rte := decodeRemoveThing(event)
err = es.handleRemoveThing(rte)
case thingDisconnect:
dte := decodeDisconnectThing(event)
err = es.handleDisconnectThing(dte)
case channelUpdate:
uce := decodeUpdateChannel(event)
err = es.handleUpdateChannel(uce)
case channelRemove:
rce := decodeRemoveChannel(event)
err = es.handleRemoveChannel(rce)
}
if err != nil {
es.logger.Warn(fmt.Sprintf("Failed to handle event sourcing: %s", err.Error()))
break
}
es.client.XAck(stream, group, msg.ID)
}
}
}
func decodeRemoveThing(event map[string]interface{}) removeEvent {
return removeEvent{
id: read(event, "id", ""),
}
}
func decodeUpdateChannel(event map[string]interface{}) updateChannelEvent {
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
strmeta := read(event, "metadata", "{}")
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(strmeta), metadata); err != nil {
metadata = map[string]interface{}{}
}
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
return updateChannelEvent{
id: read(event, "id", ""),
name: read(event, "name", ""),
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
metadata: metadata,
MF-552 - Use event sourcing to keep Bootstrap service in sync with Things service (#603) * Use separate table for Channels Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add inital event sourcing subscription Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel update sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Thing remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add Channel remove sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update service add method marshalling metadata Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Make separate methods for connection update Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Add diconnect event sync Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Configs repository mock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix service tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update repository tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update API docs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Location header Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update README.md Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix tests mutex lock Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix method names in logs Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2019-03-04 17:41:38 +01:00
}
}
func decodeRemoveChannel(event map[string]interface{}) removeEvent {
return removeEvent{
id: read(event, "id", ""),
}
}
func decodeDisconnectThing(event map[string]interface{}) disconnectEvent {
return disconnectEvent{
channelID: read(event, "chan_id", ""),
thingID: read(event, "thing_id", ""),
}
}
func (es eventStore) handleRemoveThing(rte removeEvent) error {
return es.svc.RemoveConfigHandler(rte.id)
}
func (es eventStore) handleUpdateChannel(uce updateChannelEvent) error {
channel := bootstrap.Channel{
ID: uce.id,
Name: uce.name,
Metadata: uce.metadata,
}
return es.svc.UpdateChannelHandler(channel)
}
func (es eventStore) handleRemoveChannel(rce removeEvent) error {
return es.svc.RemoveChannelHandler(rce.id)
}
func (es eventStore) handleDisconnectThing(dte disconnectEvent) error {
return es.svc.DisconnectThingHandler(dte.channelID, dte.thingID)
}
func read(event map[string]interface{}, key, def string) string {
val, ok := event[key].(string)
if !ok {
return def
}
return val
}