1
0
mirror of https://github.com/mainflux/mainflux.git synced 2025-05-01 13:48:56 +08:00

151 lines
3.6 KiB
Go
Raw Normal View History

// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package mongodb
import (
"context"
"encoding/json"
"github.com/mainflux/mainflux/pkg/errors"
"github.com/mainflux/mainflux/pkg/transformers/senml"
"github.com/mainflux/mainflux/readers"
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
const (
format = "format"
// Collection for SenML messages
defCollection = "messages"
)
var _ readers.MessageRepository = (*mongoRepository)(nil)
type mongoRepository struct {
db *mongo.Database
}
// New returns new MongoDB reader.
func New(db *mongo.Database) readers.MessageRepository {
return mongoRepository{
db: db,
}
}
func (repo mongoRepository) ReadAll(chanID string, rpm readers.PageMetadata) (readers.MessagesPage, error) {
format := defCollection
order := "time"
if rpm.Format != "" && rpm.Format != defCollection {
order = "created"
format = rpm.Format
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
}
col := repo.db.Collection(format)
sortMap := map[string]interface{}{
order: -1,
}
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
// Remove format filter and format the rest properly.
filter := fmtCondition(chanID, rpm)
cursor, err := col.Find(context.Background(), filter, options.Find().SetSort(sortMap).SetLimit(int64(rpm.Limit)).SetSkip(int64(rpm.Offset)))
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
defer cursor.Close(context.Background())
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
var messages []readers.Message
switch format {
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
case defCollection:
for cursor.Next(context.Background()) {
var m senml.Message
if err := cursor.Decode(&m); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
}
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
messages = append(messages, m)
MF-407 - Values of zero are being omitted (#434) * Fix empty protobuf values Update Normalizer service and .proto files. Reader and Writer services needs to be updated due to message format change. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update HTTP adapter to use gogo protobuf Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Reader services API Update API or Reader services to match Message changes due to switching to gogo/proto. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update InfluxDB services Update InfluxDB Reader and Writer services to match new Message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update MongoDB services Update MongoDB Reader and Writer services to match new message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra services Update Cassandra Reader and Writer service to match new Message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB Reader test Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Makefile and docs accordingly Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix possible data race in InfluxDB writer Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update InfluxDB Writer tests Raise test coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB reader Fix wrong ValueSum readings. Upadete tests and raise coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update MongoDB services tests Raise test coverage for MongoDB Reader and Writer services. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Readers API tests Raise test coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Raise test coverage Update Cassandra Reader and Writer services tests. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Use gogo/protobuf in CoAP adapter Add gogo/protobuf to Gopkg.toml and update dependencies. Update Dockerfile to run `make proto`. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra Reader tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Refactor code Improve code style and comments to improve readability. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Rename Sum to SumValue Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Rename Values to Value Since message contains only single value (or possibly no value at all), `Values` name could be misleading. Rename simple double value from `Value` to `FloatValue` accordingly. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB Reader logging Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Replace exclusive if statements with switch-case Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra services tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2018-11-05 19:18:51 +01:00
}
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
default:
for cursor.Next(context.Background()) {
var m map[string]interface{}
if err := cursor.Decode(&m); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
MF-1264 - Add support for JSON readers (#1295) * MF-1254 - Create universal JSON writer (#1260) Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add JSON support to Readers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx Reader tests Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix messages format query Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Postgres reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Cassandra Readers and writers Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Mongo reader Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Extract utility method to the JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influx and Postgres count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Update JSON transformer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix Influxdb Reader total count Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Refactor init.go for Cassandra writer Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Create a Payload type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Add comments for defaults Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix variable declarations Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Replace interface{} with a new type Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Don't set channel just to overwrite it later Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Fix range search Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> * Rename Messages field Signed-off-by: dusanb94 <dusan.borovcanin@mainflux.com> Co-authored-by: Manuel Imperiale <manuel.imperiale@gmail.com>
2020-12-30 15:43:04 +01:00
}
messages = append(messages, m)
MF-407 - Values of zero are being omitted (#434) * Fix empty protobuf values Update Normalizer service and .proto files. Reader and Writer services needs to be updated due to message format change. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update HTTP adapter to use gogo protobuf Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Reader services API Update API or Reader services to match Message changes due to switching to gogo/proto. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update InfluxDB services Update InfluxDB Reader and Writer services to match new Message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update MongoDB services Update MongoDB Reader and Writer services to match new message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra services Update Cassandra Reader and Writer service to match new Message format. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB Reader test Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Makefile and docs accordingly Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix possible data race in InfluxDB writer Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update InfluxDB Writer tests Raise test coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB reader Fix wrong ValueSum readings. Upadete tests and raise coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update MongoDB services tests Raise test coverage for MongoDB Reader and Writer services. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Readers API tests Raise test coverage. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Raise test coverage Update Cassandra Reader and Writer services tests. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Use gogo/protobuf in CoAP adapter Add gogo/protobuf to Gopkg.toml and update dependencies. Update Dockerfile to run `make proto`. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra Reader tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Refactor code Improve code style and comments to improve readability. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Rename Sum to SumValue Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Rename Values to Value Since message contains only single value (or possibly no value at all), `Values` name could be misleading. Rename simple double value from `Value` to `FloatValue` accordingly. Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Fix InfluxDB Reader logging Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Replace exclusive if statements with switch-case Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com> * Update Cassandra services tests Signed-off-by: Dusan Borovcanin <dusan.borovcanin@mainflux.com>
2018-11-05 19:18:51 +01:00
}
}
total, err := col.CountDocuments(context.Background(), filter)
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
mp := readers.MessagesPage{
PageMetadata: rpm,
Total: uint64(total),
Messages: messages,
}
return mp, nil
}
MF-596 - Add subtopic to RawMessage (#642) * Commit for mainflux-596 Modified and tested: - cli - http - mqtt - normalizer - all readers - sdk messages - all writers - ws Missing: - coap - lora Signed-off-by: ale <ale@metaverso.org> * - fix subtopic name in, when starting with dot, http/ws/mqtt - add some test on readers Signed-off-by: ale <ale@metaverso.org> * - fix string concatenation - update http/transport regexp to match subtopic names with only \w- - update ws/transport regexp like http ones with also the wildcard * and > Signed-off-by: ale <ale@metaverso.org> * added subtopic support to coap adapter Signed-off-by: ale <ale@metaverso.org> * - update replace functions with replaceall when needed - renamed getDestChannel to fmtSubject - update api/transport and ws/transport route to be more readable - fix mqtt syntax - renamed func andQuery to query as suggested by @anovakovic01 - have a nice we :) Signed-off-by: ale <ale@metaverso.org> * - fix error declaration on ws/nat/publisher - fix regexp added missing allowed chars - and _ on coap/api/transport - fix subtopic clean suffix / if present on coap/api/transport - improve regexp on http and ws /api/transport, now does not accept url that do not strictly match - add some ws subtopic tests Signed-off-by: ale <ale@metaverso.org> * - enabled wildcard chars on coap/api/transport - allow use special chars on http and ws api/transport Signed-off-by: ale <ale@metaverso.org> * - use strings.Replace() insted ReplaceAll() Signed-off-by: ale <ale@metaverso.org> * - allow every chars on subtopics - fix replace error on mqtt Signed-off-by: ale <ale@metaverso.org> * fix cassandra test Signed-off-by: ale <ale@metaverso.org> * fix ws test with invalid subtopic Signed-off-by: ale <ale@metaverso.org> * fix invalid GOCACHE in go1.12, replaced by -count 1, see https://golang.org/doc/go1.10#test Signed-off-by: ale <ale@metaverso.org> * - improve regexp on http/ws api/transport - minor changes Signed-off-by: ale <ale@metaverso.org> * - add generic function parseSubtopic on ws/http adapters Signed-off-by: ale <ale@metaverso.org> * - add generic function fmtSubtopic on coap adapter Signed-off-by: ale <ale@metaverso.org>
2019-03-15 18:38:07 +01:00
func fmtCondition(chanID string, rpm readers.PageMetadata) bson.D {
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
filter := bson.D{
bson.E{
Key: "channel",
Value: chanID,
},
}
var query map[string]interface{}
meta, err := json.Marshal(rpm)
if err != nil {
return filter
}
json.Unmarshal(meta, &query)
MF-596 - Add subtopic to RawMessage (#642) * Commit for mainflux-596 Modified and tested: - cli - http - mqtt - normalizer - all readers - sdk messages - all writers - ws Missing: - coap - lora Signed-off-by: ale <ale@metaverso.org> * - fix subtopic name in, when starting with dot, http/ws/mqtt - add some test on readers Signed-off-by: ale <ale@metaverso.org> * - fix string concatenation - update http/transport regexp to match subtopic names with only \w- - update ws/transport regexp like http ones with also the wildcard * and > Signed-off-by: ale <ale@metaverso.org> * added subtopic support to coap adapter Signed-off-by: ale <ale@metaverso.org> * - update replace functions with replaceall when needed - renamed getDestChannel to fmtSubject - update api/transport and ws/transport route to be more readable - fix mqtt syntax - renamed func andQuery to query as suggested by @anovakovic01 - have a nice we :) Signed-off-by: ale <ale@metaverso.org> * - fix error declaration on ws/nat/publisher - fix regexp added missing allowed chars - and _ on coap/api/transport - fix subtopic clean suffix / if present on coap/api/transport - improve regexp on http and ws /api/transport, now does not accept url that do not strictly match - add some ws subtopic tests Signed-off-by: ale <ale@metaverso.org> * - enabled wildcard chars on coap/api/transport - allow use special chars on http and ws api/transport Signed-off-by: ale <ale@metaverso.org> * - use strings.Replace() insted ReplaceAll() Signed-off-by: ale <ale@metaverso.org> * - allow every chars on subtopics - fix replace error on mqtt Signed-off-by: ale <ale@metaverso.org> * fix cassandra test Signed-off-by: ale <ale@metaverso.org> * fix ws test with invalid subtopic Signed-off-by: ale <ale@metaverso.org> * fix invalid GOCACHE in go1.12, replaced by -count 1, see https://golang.org/doc/go1.10#test Signed-off-by: ale <ale@metaverso.org> * - improve regexp on http/ws api/transport - minor changes Signed-off-by: ale <ale@metaverso.org> * - add generic function parseSubtopic on ws/http adapters Signed-off-by: ale <ale@metaverso.org> * - add generic function fmtSubtopic on coap adapter Signed-off-by: ale <ale@metaverso.org>
2019-03-15 18:38:07 +01:00
for name, value := range query {
switch name {
case
"channel",
"subtopic",
"publisher",
"name",
"protocol":
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
filter = append(filter, bson.E{Key: name, Value: value})
case "v":
bsonFilter := value
val, ok := query["comparator"]
if ok {
switch val.(string) {
case readers.EqualKey:
bsonFilter = value
case readers.LowerThanKey:
bsonFilter = bson.M{"$lt": value}
case readers.LowerThanEqualKey:
bsonFilter = bson.M{"$lte": value}
case readers.GreaterThanKey:
bsonFilter = bson.M{"$gt": value}
case readers.GreaterThanEqualKey:
bsonFilter = bson.M{"$gte": value}
}
}
filter = append(filter, bson.E{Key: "value", Value: bsonFilter})
case "vb":
filter = append(filter, bson.E{Key: "bool_value", Value: value})
case "vs":
filter = append(filter, bson.E{Key: "string_value", Value: value})
case "vd":
filter = append(filter, bson.E{Key: "data_value", Value: value})
case "from":
filter = append(filter, bson.E{Key: "time", Value: bson.M{"$gte": value}})
case "to":
filter = append(filter, bson.E{Key: "time", Value: bson.M{"$lt": value}})
MF-596 - Add subtopic to RawMessage (#642) * Commit for mainflux-596 Modified and tested: - cli - http - mqtt - normalizer - all readers - sdk messages - all writers - ws Missing: - coap - lora Signed-off-by: ale <ale@metaverso.org> * - fix subtopic name in, when starting with dot, http/ws/mqtt - add some test on readers Signed-off-by: ale <ale@metaverso.org> * - fix string concatenation - update http/transport regexp to match subtopic names with only \w- - update ws/transport regexp like http ones with also the wildcard * and > Signed-off-by: ale <ale@metaverso.org> * added subtopic support to coap adapter Signed-off-by: ale <ale@metaverso.org> * - update replace functions with replaceall when needed - renamed getDestChannel to fmtSubject - update api/transport and ws/transport route to be more readable - fix mqtt syntax - renamed func andQuery to query as suggested by @anovakovic01 - have a nice we :) Signed-off-by: ale <ale@metaverso.org> * - fix error declaration on ws/nat/publisher - fix regexp added missing allowed chars - and _ on coap/api/transport - fix subtopic clean suffix / if present on coap/api/transport - improve regexp on http and ws /api/transport, now does not accept url that do not strictly match - add some ws subtopic tests Signed-off-by: ale <ale@metaverso.org> * - enabled wildcard chars on coap/api/transport - allow use special chars on http and ws api/transport Signed-off-by: ale <ale@metaverso.org> * - use strings.Replace() insted ReplaceAll() Signed-off-by: ale <ale@metaverso.org> * - allow every chars on subtopics - fix replace error on mqtt Signed-off-by: ale <ale@metaverso.org> * fix cassandra test Signed-off-by: ale <ale@metaverso.org> * fix ws test with invalid subtopic Signed-off-by: ale <ale@metaverso.org> * fix invalid GOCACHE in go1.12, replaced by -count 1, see https://golang.org/doc/go1.10#test Signed-off-by: ale <ale@metaverso.org> * - improve regexp on http/ws api/transport - minor changes Signed-off-by: ale <ale@metaverso.org> * - add generic function parseSubtopic on ws/http adapters Signed-off-by: ale <ale@metaverso.org> * - add generic function fmtSubtopic on coap adapter Signed-off-by: ale <ale@metaverso.org>
2019-03-15 18:38:07 +01:00
}
}
MF-549 - Change metadata format from JSON string to JSON object (#706) * Update metadata type in things service Update things service so that metadata has map type. Update repo implementation by adding sqlx lib. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add sqlx lib to bootstrap service Add sqlx lib to bootstrap service and update metadata field type. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update metadata in redis streams consumer Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update tests for bootstrap service Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader logging and driver version Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix mongo reader and writer Fix mongo reader and writer by updating driver version. Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update SDK with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update LoRa adapter with new metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update users service in order to use sqlx Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Replace anonymous struct with map Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Update docs for LoRa adapter Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix LoRa application metadata format Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Fix metadata format in LoRa docs Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com> * Add metadata2 var to SDK things test Signed-off-by: Aleksandar Novakovic <aleksandar.novakovic@mainflux.com>
2019-04-16 14:58:56 +02:00
return filter
MF-596 - Add subtopic to RawMessage (#642) * Commit for mainflux-596 Modified and tested: - cli - http - mqtt - normalizer - all readers - sdk messages - all writers - ws Missing: - coap - lora Signed-off-by: ale <ale@metaverso.org> * - fix subtopic name in, when starting with dot, http/ws/mqtt - add some test on readers Signed-off-by: ale <ale@metaverso.org> * - fix string concatenation - update http/transport regexp to match subtopic names with only \w- - update ws/transport regexp like http ones with also the wildcard * and > Signed-off-by: ale <ale@metaverso.org> * added subtopic support to coap adapter Signed-off-by: ale <ale@metaverso.org> * - update replace functions with replaceall when needed - renamed getDestChannel to fmtSubject - update api/transport and ws/transport route to be more readable - fix mqtt syntax - renamed func andQuery to query as suggested by @anovakovic01 - have a nice we :) Signed-off-by: ale <ale@metaverso.org> * - fix error declaration on ws/nat/publisher - fix regexp added missing allowed chars - and _ on coap/api/transport - fix subtopic clean suffix / if present on coap/api/transport - improve regexp on http and ws /api/transport, now does not accept url that do not strictly match - add some ws subtopic tests Signed-off-by: ale <ale@metaverso.org> * - enabled wildcard chars on coap/api/transport - allow use special chars on http and ws api/transport Signed-off-by: ale <ale@metaverso.org> * - use strings.Replace() insted ReplaceAll() Signed-off-by: ale <ale@metaverso.org> * - allow every chars on subtopics - fix replace error on mqtt Signed-off-by: ale <ale@metaverso.org> * fix cassandra test Signed-off-by: ale <ale@metaverso.org> * fix ws test with invalid subtopic Signed-off-by: ale <ale@metaverso.org> * fix invalid GOCACHE in go1.12, replaced by -count 1, see https://golang.org/doc/go1.10#test Signed-off-by: ale <ale@metaverso.org> * - improve regexp on http/ws api/transport - minor changes Signed-off-by: ale <ale@metaverso.org> * - add generic function parseSubtopic on ws/http adapters Signed-off-by: ale <ale@metaverso.org> * - add generic function fmtSubtopic on coap adapter Signed-off-by: ale <ale@metaverso.org>
2019-03-15 18:38:07 +01:00
}