1
0
mirror of https://github.com/mainflux/mainflux.git synced 2025-05-04 22:17:59 +08:00
Aryan Godara 986edacfc7
MF-1584 - Upgrade InfluxDB from 1.x to 2.x (#1709)
* Upgrade InfluxDB from 1.x to 2.x

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influx DB configuration updated

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connection to InfluxDBv2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Token cannot be created

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connected to InfluxDB2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connected to InfluxDB2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB v2 Consumer Implementation

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* quickfix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb-Writer Unit Tests Update

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Consumer Update

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB Writer Tests Implemented

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB Connection Check

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolving Remarks

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved consumer-test remark

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* consumer-test slow working version

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* reader changes

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Consumer tests time issue fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Eof warning fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Makefile Fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Readers Initial Setup

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* consumer json fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Bugfix and resolves comments.

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* one test fails

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* solved last page read

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* writers future time problem fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* weird

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* weird

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Tests Passes

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolve Semaphore Issues

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* UUID comment on consumer tests resolved

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Inclusive from and Exclusive to

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Inclusive from and Exclusive to

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* 1 second limits

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* 1 second limits

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fixed json time

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fixed CI error

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved request

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved request

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved requests

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* removed blank line

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved comment

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* deleted unnecessary string builder

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* initial commit

Signed-off-by: aryan <aryangodara03@gmail.com>

* change influxdb docker image version.

Signed-off-by: aryan <aryangodara03@gmail.com>

* go mod and vendor fixing

Signed-off-by: aryan <aryangodara03@gmail.com>

* Upgrade InfluxDB from 1.x to 2.x

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influx DB configuration updated

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connection to InfluxDBv2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Token cannot be created

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connected to InfluxDB2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Connected to InfluxDB2

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB v2 Consumer Implementation

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* quickfix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb-Writer Unit Tests Update

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Consumer Update

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB Writer Tests Implemented

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* InfluxDB Connection Check

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolving Remarks

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved consumer-test remark

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* consumer-test slow working version

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* reader changes

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Consumer tests time issue fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolved Reviews

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Readers Initial Setup

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* consumer json fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* readers simple version fix

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Influxdb Reader Parsers

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Bugfix and resolves comments.

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* one test fails

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* solved last page read

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* writers future time problem fixed

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* weird

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* weird

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Tests Passes

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Resolve Semaphore Issues

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* UUID comment on consumer tests resolved

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Inclusive from and Exclusive to

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* Inclusive from and Exclusive to

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* 1 second limits

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* 1 second limits

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved review

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fixed json time

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* fixed CI error

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved request

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved request

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved requests

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* removed blank line

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* resolved comment

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* deleted unnecessary string builder

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>

* initial commit

Signed-off-by: aryan <aryangodara03@gmail.com>

* change influxdb docker image version.

Signed-off-by: aryan <aryangodara03@gmail.com>

* go mod and vendor fixing

Signed-off-by: aryan <aryangodara03@gmail.com>

* go mod fixing

Signed-off-by: aryan <aryangodara03@gmail.com>

* make tests pass locally, fix errors

Signed-off-by: aryan <aryangodara03@gmail.com>

* rem unsused variables/consts

Signed-off-by: aryan <aryangodara03@gmail.com>

* go mod tidy vendor

Signed-off-by: aryan <aryangodara03@gmail.com>

* add env var and data types

Signed-off-by: aryan <aryangodara03@gmail.com>

* update influxdb version to latest (2.3 to 2.12)

Signed-off-by: aryan <aryangodara03@gmail.com>

* change time precision for message tests

Signed-off-by: aryan <aryangodara03@gmail.com>

* renamed influxdb2 to influxdata

Signed-off-by: aryan <aryangodara03@gmail.com>

* address remarks on PR

Signed-off-by: aryan <aryangodara03@gmail.com>

* update influxbd according to latest mf commit

Signed-off-by: aryan <aryangodara03@gmail.com>

* temp commit, rem before pushing

Signed-off-by: aryan <aryangodara03@gmail.com>

* update main files and cassandra-reader messages_tests

Signed-off-by: aryan <aryangodara03@gmail.com>

* fix name of logger while importing

Signed-off-by: aryan <aryangodara03@gmail.com>

* remove unnecessary print lines

Signed-off-by: aryan <aryangodara03@gmail.com>

* correct env var name

Signed-off-by: aryan <aryangodara03@gmail.com>

* change to async consume

Signed-off-by: aryan <aryangodara03@gmail.com>

* add option to switch bw sync and async

Signed-off-by: aryan <aryangodara03@gmail.com>

* test for both async and sync

Signed-off-by: aryan <aryangodara03@gmail.com>

* update consumer and add writeAPIs to config

Signed-off-by: aryan <aryangodara03@gmail.com>

* revert back to sync consuming

Signed-off-by: aryan <aryangodara03@gmail.com>

* temp fix for default timeout value

Signed-off-by: aryan <aryangodara03@gmail.com>

* set default timeout in config.

Signed-off-by: aryan <aryangodara03@gmail.com>

* remove unwanted env vars, add required ones.

Signed-off-by: aryan <aryangodara03@gmail.com>

* rem unused username password from config

Signed-off-by: aryan <aryangodara03@gmail.com>

* update readme, env vars, and remove grafana

Signed-off-by: aryan <aryangodara03@gmail.com>

* update readme

Signed-off-by: aryan <aryangodara03@gmail.com>

* fix typo

Signed-off-by: aryan <aryangodara03@gmail.com>

* update readme description.

Signed-off-by: aryan <aryangodara03@gmail.com>

* fix more typos.

Signed-off-by: aryan <aryangodara03@gmail.com>

* add link to official docs to readme.

Signed-off-by: aryan <aryangodara03@gmail.com>

---------

Signed-off-by: fatih <fatihdurmaz@sabanciuniv.edu>
Signed-off-by: aryan <aryangodara03@gmail.com>
Co-authored-by: fatih <fatihdurmaz@sabanciuniv.edu>
Co-authored-by: Dušan Borovčanin <dusan.borovcanin@mainflux.com>
2023-03-16 02:36:14 +01:00

193 lines
6.1 KiB
Go

// Copyright 2020-2021 InfluxData, Inc. All rights reserved.
// Use of this source code is governed by MIT
// license that can be found in the LICENSE file.
// Package http provides HTTP servicing related code.
//
// Important type is Service which handles HTTP operations. It is internally used by library and it is not necessary to use it directly for common operations.
// It can be useful when creating custom InfluxDB2 server API calls using generated code from the domain package, that are not yet exposed by API of this library.
//
// Service can be obtained from client using HTTPService() method.
// It can be also created directly. To instantiate a Service use NewService(). Remember, the authorization param is in form "Token your-auth-token". e.g. "Token DXnd7annkGteV5Wqx9G3YjO9Ezkw87nHk8OabcyHCxF5451kdBV0Ag2cG7OmZZgCUTHroagUPdxbuoyen6TSPw==".
// srv := http.NewService("http://localhost:8086", "Token my-token", http.DefaultOptions())
package http
import (
"context"
"encoding/json"
"io"
"io/ioutil"
"mime"
"net/http"
"net/url"
"strconv"
http2 "github.com/influxdata/influxdb-client-go/v2/internal/http"
"github.com/influxdata/influxdb-client-go/v2/internal/log"
)
// RequestCallback defines function called after a request is created before any call
type RequestCallback func(req *http.Request)
// ResponseCallback defines function called after a successful response was received
type ResponseCallback func(resp *http.Response) error
// Service handles HTTP operations with taking care of mandatory request headers and known errors
type Service interface {
// DoPostRequest sends HTTP POST request to the given url with body
DoPostRequest(ctx context.Context, url string, body io.Reader, requestCallback RequestCallback, responseCallback ResponseCallback) *Error
// DoHTTPRequest sends given HTTP request and handles response
DoHTTPRequest(req *http.Request, requestCallback RequestCallback, responseCallback ResponseCallback) *Error
// DoHTTPRequestWithResponse sends given HTTP request and returns response
DoHTTPRequestWithResponse(req *http.Request, requestCallback RequestCallback) (*http.Response, error)
// SetAuthorization sets the authorization header value
SetAuthorization(authorization string)
// Authorization returns current authorization header value
Authorization() string
// ServerAPIURL returns URL to InfluxDB2 server API space
ServerAPIURL() string
// ServerURL returns URL to InfluxDB2 server
ServerURL() string
}
// service implements Service interface
type service struct {
serverAPIURL string
serverURL string
authorization string
client Doer
userAgent string
}
// NewService creates instance of http Service with given parameters
func NewService(serverURL, authorization string, httpOptions *Options) Service {
apiURL, err := url.Parse(serverURL)
serverAPIURL := serverURL
if err == nil {
apiURL, err = apiURL.Parse("api/v2/")
if err == nil {
serverAPIURL = apiURL.String()
}
}
return &service{
serverAPIURL: serverAPIURL,
serverURL: serverURL,
authorization: authorization,
client: httpOptions.HTTPDoer(),
userAgent: http2.FormatUserAgent(httpOptions.ApplicationName()),
}
}
func (s *service) ServerAPIURL() string {
return s.serverAPIURL
}
func (s *service) ServerURL() string {
return s.serverURL
}
func (s *service) SetAuthorization(authorization string) {
s.authorization = authorization
}
func (s *service) Authorization() string {
return s.authorization
}
func (s *service) DoPostRequest(ctx context.Context, url string, body io.Reader, requestCallback RequestCallback, responseCallback ResponseCallback) *Error {
return s.doHTTPRequestWithURL(ctx, http.MethodPost, url, body, requestCallback, responseCallback)
}
func (s *service) doHTTPRequestWithURL(ctx context.Context, method, url string, body io.Reader, requestCallback RequestCallback, responseCallback ResponseCallback) *Error {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return NewError(err)
}
return s.DoHTTPRequest(req, requestCallback, responseCallback)
}
func (s *service) DoHTTPRequest(req *http.Request, requestCallback RequestCallback, responseCallback ResponseCallback) *Error {
resp, err := s.DoHTTPRequestWithResponse(req, requestCallback)
if err != nil {
return NewError(err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return s.parseHTTPError(resp)
}
if responseCallback != nil {
err := responseCallback(resp)
if err != nil {
return NewError(err)
}
}
return nil
}
func (s *service) DoHTTPRequestWithResponse(req *http.Request, requestCallback RequestCallback) (*http.Response, error) {
log.Infof("HTTP %s req to %s", req.Method, req.URL.String())
if len(s.authorization) > 0 {
req.Header.Set("Authorization", s.authorization)
}
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", s.userAgent)
}
if requestCallback != nil {
requestCallback(req)
}
return s.client.Do(req)
}
func (s *service) parseHTTPError(r *http.Response) *Error {
// successful status code range
if r.StatusCode >= 200 && r.StatusCode < 300 {
return nil
}
defer func() {
// discard body so connection can be reused
_, _ = io.Copy(ioutil.Discard, r.Body)
_ = r.Body.Close()
}()
perror := NewError(nil)
perror.StatusCode = r.StatusCode
if v := r.Header.Get("Retry-After"); v != "" {
r, err := strconv.ParseUint(v, 10, 32)
if err == nil {
perror.RetryAfter = uint(r)
}
}
// json encoded error
ctype, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if ctype == "application/json" {
perror.Err = json.NewDecoder(r.Body).Decode(perror)
} else {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
perror.Err = err
return perror
}
perror.Code = r.Status
perror.Message = string(body)
}
if perror.Code == "" && perror.Message == "" {
switch r.StatusCode {
case http.StatusTooManyRequests:
perror.Code = "too many requests"
perror.Message = "exceeded rate limit"
case http.StatusServiceUnavailable:
perror.Code = "unavailable"
perror.Message = "service temporarily unavailable"
default:
perror.Code = r.Status
perror.Message = r.Header.Get("X-Influxdb-Error")
}
}
return perror
}