Learn Golang scenario-based solutions in CSDN (grpc-based microservice development scaffolding)

One, use TLS encrypted communication between services

In Golang’s gRPC-based microservice development, TLS encrypted communication can be used to ensure secure communication between services. Here is a simple design example:

  1. Generate certificate and key:
$ openssl req -newkey rsa:2048 -nodes -keyout server.key \
    -x509 -days 365 -out server.crt
  1. Define the gRPC server:
func newServer() (*grpc. Server, error) {
// load the certificate and key
creds, err := credentials.NewServerTLSFromFile("server.crt", "server.key")
if err != nil {
return nil, err
}

// Create gRPC server and add certificate and interceptor
srv := grpc.NewServer(
        grpc.Creds(creds),
        grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
            // Add other middleware interceptors, such as authentication, logging, etc.
        )),
    )

    // Register gRPC service
    pb.RegisterUserServiceServer(srv, & userService{})

return srv, nil
}
  1. The client connects to the gRPC server:
func main() {
    // load the certificate and key, and create a credential object
    creds, err := credentials.NewClientTLSFromFile("server.crt", "")
    if err != nil {
        log. Fatal(err)
    }

    // Establish a connection and create a client object, and add an interceptor (optional)
    conn, err := grpc.Dial(":9000", grpc.WithTransportCredentials(creds))
    if err != nil {
        log. Fatal(err)
    }
    
    defer conn. Close()

client := pb.NewUserServiceClient(conn)

    //...
}
  1. Add TLS encrypted communication in the service implementation:
type userService struct {
pb. UnimplementedUserServiceServer
}

func (s *userService) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
// Get the user ID from the request and query user information
user := &User{ID: req.Id}
err := db.First(user).Error
if err != nil {
return nil, status.Errorf(codes.NotFound, "User not found")
}

// Convert user information to gRPC response object and return
res := &pb. GetUserResponse{
Id: user.ID,
Name: user.Name,
Email: user. Email,
}

return res, nil
}

func newServer() (*grpc. Server, error) {
// load the certificate and key, and create a credential object
creds, err := credentials.NewServerTLSFromFile("server.crt", "server.key")
if err != nil {
return nil, err
}

// Create gRPC server and add certificate and interceptor
srv := grpc.NewServer(
        grpc.Creds(creds),
        grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
            // Add other middleware interceptors, such as authentication, logging, etc.
        )),
    )

    // Register gRPC service
    pb.RegisterUserServiceServer(srv, & userService{})

return srv, nil
}

In this way, in Golang’s gRPC-based microservice development, TLS encrypted communication can be used to ensure secure communication between services. Note, when generating certificates and keys, please replace them with your own certificates and keys according to the actual situation.

Second, etcd service registration and service discovery

In Golang’s gRPC-based microservice development, etcd can be used to implement service registration and service discovery. Here is a simple design example:

  1. Install etcd client:
$ go get go.etcd.io/etcd/clientv3
  1. Register with etcd on service startup:
func main() {
    //...
    
// create etcd client and connect to etcd server
cli, err := clientv3.New(clientv3.Config{
Endpoints: []string{"localhost:2379"},
DialTimeout: 5 * time. Second,
})
if err != nil {
log. Fatal(err)
}
defer cli. Close()

// Create gRPC server and add certificate and interceptor
srv := grpc.NewServer(
        //...
    )

    // Register gRPC service
    pb.RegisterUserServiceServer(srv, & userService{})

    // Start the gRPC server
go func() {
lis, err := net.Listen("tcp", ":9000")
if err != nil {
log. Fatal(err)
}

        if err = srv. Serve(lis); err != nil {
            log.Fatalf("failed to serve: %v", err)
        }
    }()

    // Register service information (IP address, port, etc.) in etcd
serviceKey := fmt.Sprintf("/services/%s/%s:%d",
                            serviceName, serviceIP, servicePort)

resp, err := cli.Grant(context.Background(), 5*time.Minute)
if err != nil {
log. Fatal(err)
}

if _, err = cli.Put(context.Background(), serviceKey, "", clientv3.WithLease(resp.ID));
        if err != nil {
            log. Fatal(err)
        }
    
    //...
}
  1. Obtain the service address from etcd in the client:
func main() {
    //...

// create etcd client and connect to etcd server
cli, err := clientv3.New(clientv3.Config{
Endpoints: []string{"localhost:2379"},
DialTimeout: 5 * time. Second,
})
if err != nil {
log. Fatal(err)
}
defer cli. Close()

    // Get service address from etcd
serviceKey := fmt.Sprintf("/services/%s", serviceName)

resp, err := cli.Get(context.Background(), serviceKey, clientv3.WithPrefix())
if err != nil {
log. Fatal(err)
}

var addresses [] string
for _, kv := range resp. Kvs {
address := string(kv.Key)[len(serviceKey) + 1:] // remove the prefix
addresses = append(addresses, address)
}

    // Create gRPC connection and client objects, and add interceptors (optional)
conn, err := grpc.Dial(addresses[rand.Int()%len(addresses)],
            grpc.WithTransportCredentials(creds),
            grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(
                // Add other middleware interceptors, such as authentication, logging, etc.
            )),
        )
if err != nil {
log. Fatal(err)
}
defer conn. Close()

client := pb.NewUserServiceClient(conn)

    //...
}

In this way, in Golang’s gRPC-based microservice development, etcd can be used to implement service registration and service discovery. Note that when registering to etcd when the service starts, you need to replace it with your own IP address and port number, and modify the service name and etcd server address according to the actual situation. When obtaining the service address from etcd in the client, you need to replace it with your own service name according to the actual situation.

Three, etcd application configuration center

In Golang’s gRPC-based microservice development, etcd can be used as the application configuration center. Here is a simple design example:

  1. Install etcd client:
$ go get go.etcd.io/etcd/clientv3
  1. Load configuration from etcd on service startup:
import (
"context"
"encoding/json"
"fmt"
"log"
"time"

clientv3 "go.etcd.io/etcd/client/v3"
)

type Config struct {
    // configuration item structure definition
}

func main() {
    //...

    // create etcd client and connect to etcd server
cli, err := clientv3.New(clientv3.Config{
Endpoints: []string{"localhost:2379"},
DialTimeout: 5 * time. Second,
})
if err != nil {
log. Fatal(err)
}
defer cli. Close()

    // Get configuration information from etcd
configKey := fmt.Sprintf("/configs/%s", serviceName)

resp, err := cli. Get(context. Background(), configKey)
if err != nil {
log. Fatal(err)
}

var config Config
for _, kv := range resp. Kvs {
if err = json.Unmarshal(kv.Value, & amp;config); err != nil {
log.Fatalf("failed to unmarshal config from etcd: %v", err)
}
}

    // Use configuration items to initialize other components, objects, etc.

    //...
}
  1. Monitor configuration changes in etcd at runtime:
import (
clientv3 "go.etcd.io/etcd/client/v3"
)

func watchConfig(cli *clientv3.Client, serviceName string) (<-chan *Config, error) {
configKey := fmt.Sprintf("/configs/%s", serviceName)

// Create Watcher
watcher := clientv3.NewWatcher(cli)
defer watcher. Close()

// Monitor the change of configKey and return the updated configuration item
ch := make(chan *Config, 1)
go func() {
var config Config

for {
resp, err := cli. Get(context. Background(), configKey)
if err != nil {
log.Printf("failed to get config from etcd: %v", err)
continue
}

if len(resp. Kvs) == 0 {
log.Println("no configuration found")
continue
}

if err = json.Unmarshal(resp.Kvs[0].Value, & amp;config); err != nil {
log.Printf("failed to unmarshal config from etcd: %v", err)
continue
}

            // Send new configuration items to ch
select {
case ch <- &config:
default:
}
}
}()

return ch, nil
}

In this way, in the development of microservices based on gRPC in Golang, etcd can be used as the application configuration center. Note that when loading the configuration from etcd when the service starts, you need to modify the service name and etcd server address according to the actual situation. When monitoring configuration changes in etcd at runtime, the return value ch needs to be passed to other components, objects, etc., so that they can reinitialize themselves when the configuration changes.

Fourth, EFK unified log collection

In Golang’s gRPC-based microservice development, EFK (Elasticsearch + Fluentd + Kibana) can be used as a unified log collection solution. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go
  1. Integrate gRPC-Go and Logrus:
import (
"context"
"log"

"google.golang.org/grpc"
"github.com/sirupsen/logrus"
)

func main() {
    //...

    // Use Logrus to record logs and configure them to be sent to Fluentd
logrus. SetFormatter( & logrus. JSONFormatter{})
logrus. SetLevel(logrus. InfoLevel)
logrus.SetOutput(fluentHook{host: "localhost", port: 24224})

    //...
}

// Define a Logrus Hook for sending logs to Fluentd
type fluentHook struct {
host string
port int
}

func (hook fluentHook) Fire(entry *logrus.Entry) error {
tag := entry.Logger.Out.(*fluentLogger).tag

event := map[string]interface{}{
"message": entry. Message,
}
for k, v := range entry. Data {
event[k] = v
}

if _, err := fluent.New().PostWithTime(tag, time.Now(), event); err != nil {
log.Printf("failed to send log to fluentd: %v", err)
        return err
    }
return nil
}

func (hook fluentHook) Levels() [] logrus. Level {
return logrus. AllLevels[:]
}

// Define a Fluentd Logger that implements the io.Writer interface
type fluentLogger struct {
tag string
}

func (logger *fluentLogger) Write(p []byte) (int, error) {
event := map[string]interface{}{
"message": string(p),
}
if _, err := fluent.New().PostWithTime(logger.tag, time.Now(), event); err != nil {
log.Printf("failed to send log to fluentd: %v", err)
        return 0, err
    }
return len(p), nil
}
  1. Start EFK in Docker Compose:
version: '3'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
    environment:
      - discovery.type=single-node

  kibana:
    image: docker.elastic.co/kibana/kibana:7.14.1
    ports:
      - "5601:5601"
    depends_on:
      -elasticsearch

  fluentd:
    image: fluent/fluentd:v1.12-1
    volumes:
      - ./conf:/fluentd/etc # Configuration file directory, including fluent.conf and pos_file directory, etc.
      - /var/log:/var/log # log file directory, used to collect system logs and container logs, etc.
  1. Configure the Elasticsearch output plugin in Fluentd:
<match **>
  @type elasticsearch_dynamic_buffered
  hosts elasticsearch:9200

  include_tag_keytrue
  tag_key@log_name

  buffer_type file # Use the file cache method, of course, you can also use memory or redis.
  buffer_path /var/log/td-agent/buffer/out_*.log # The path of the cache file, which can be modified according to the actual situation
  buffer_chunk_limit 1m # cache file size limit
  buffer_queue_limit 100 # buffer queue length limit

  logstash_format true # Use logstash format to output to Elasticsearch
</match>
  1. Use Logrus to record logs in Golang gRPC-based microservice development:
import (
    "github.com/sirupsen/logrus"
)

func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
    logrus.WithFields(logrus.Fields{
        "name": in.Name,
        "age": in. Age,
    }).Info("received a request")
    
    //...

    return &pb.HelloReply{Message: "Hello " + in.Name}, nil
}

In this way, in Golang’s gRPC-based microservice development, EFK can be used as a unified log collection solution. Note that when integrating gRPC-Go and Logrus, you need to configure Logrus Hook before log output and send logs to Fluentd. When starting EFK in Docker Compose, you need to start the three containers of Elasticsearch, Kibana and Fluentd respectively, and configure the Elasticsearch output plug-in in Fluentd. Finally, using Logrus to record logs in Golang’s gRPC-based microservice development can be automatically sent to EFK for unified management and query.

Five, viper configuration file read

In Golang’s gRPC-based microservice development, Viper can be used as a configuration file reading solution. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go

# install viper
$ go get github.com/spf13/viper
  1. Create a config directory in the project root directory and add the default configuration file config.yaml:
server:
  port: 8080

database:
  host: localhost
  port: 3306
  user: root
  password: root1234
  database: testdb

logger:
  level: debug
  1. Initialize Viper in the main function and read the configuration file:
import (
"log"

"github.com/spf13/viper"
)

func main() {
    // Initialize Viper, and set parameters such as default value and search path.
v := viper. New()
v. SetConfigName("config")
v.AddConfigPath("./config")
v.SetDefault("server.port", "8080")
v.SetDefault("database.host", "localhost")
v.SetDefault("database.port", "3306")
v.SetDefault("database.user", "root")
v.SetDefault("database.password", "")
v.SetDefault("database.database", "testdb")

// If the environment variable CONFIG_PATH exists, add it to the search path.
if path := os. Getenv("CONFIG_PATH"); path != "" {
log.Printf("adding config search path %s\\
", path)
v.AddConfigPath(path)
}

    // Perform initialization work according to the above configuration information.
if err := v.ReadInConfig(); err != nil {
log.Fatalf("failed to read config file: %v", err)
}
    //...
}
  1. Use Viper to read configuration files in Golang gRPC-based microservice development:
import (
"github.com/spf13/viper"
)

type server struct {
port string
}

func main() {
    //...

    // Read the server.port parameter from the configuration file, if not set, use the default value 8080.
s := &server{port: viper. GetString("server.port")}

    //...
}

In this way, in Golang’s gRPC-based microservice development, Viper can be used as a configuration file reading solution. Note that Viper needs to be initialized in the main function, and parameters such as default values and search paths need to be set, and then initialized according to the above configuration information. When using Viper to read configuration files in Golang’s gRPC-based microservice development, you only need to call methods such as viper.GetString to obtain the specified parameter values.

Six, logurs log component encapsulation

In Golang gRPC-based microservice development, logrus can be used as a log component. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go

# Install logrus and the json format output plugin of logrus.
$ go get github.com/sirupsen/logrus
$ go get github.com/mattn/go-isatty
  1. Initialize Logrus in the main function and set parameters such as log level and output format:
import (
"log"
"os"

"github.com/sirupsen/logrus"
"github.com/mattn/go-isatty"
)

func main() {
    // Initialize Logrus, and set log level, output format and other parameters.
logrus. SetFormatter( & logrus. JSONFormatter{})
if isatty. IsTerminal(os. Stdout. Fd()) {
logrus. SetLevel(logrus. DebugLevel)
} else {
logrus. SetLevel(logrus. InfoLevel)
}
logrus. SetOutput(os. Stdout)

    //...
}
  1. Encapsulate Logrus for use in Golang gRPC-based microservice development:
import (
    "github.com/sirupsen/logrus"
)

var log *logrus.Entry

func init() {
    log = logrus.WithFields(logrus.Fields{
        "app": "myapp",
        "env": "production",
    })
}

func doSomething() {
log.Debugf("debug message")
log.Infof("info message")
log.Warnf("warning message")
log.Errorf("error message")
}

In this way, in Golang’s gRPC-based microservice development, Logrus can be used as a log component. Note that Logrus needs to be initialized in the main function, and parameters such as log level and output format need to be set. When encapsulating Logrus, you can create a global log instance in the init function, set some default field values, and then call the corresponding method where the log needs to be printed. This method avoids repeatedly creating log instances and ensures that all logs have the same field values.

Golang cloud-native learning roadmap, teaching videos, documents, interview questions (materials include C/C ++, K8s, golang project combat, gRPC, Docker, DevOps, etc.) for free sharing. If you need it, you can add qun: 793221798 to receive

Seven, distributed log link tracking design

In Golang’s gRPC-based microservice development, it is often necessary to log and link trace distributed systems for better monitoring and debugging. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go

# Install logrus and the json format output plugin of logrus.
$ go get github.com/sirupsen/logrus
$ go get github.com/mattn/go-isatty

# Install opentracing related libraries.
$ go get github.com/opentracing/opentracing-go
$ go get github.com/uber/jaeger-client-go/config
  1. In the main function, initialize components such as Logrus and Jaeger and set parameters such as log level and output format:
import (
"log"
"os"

"github.com/sirupsen/logrus"
"github.com/mattn/go-isatty"

    "github.com/opentracing/opentracing-go"
    "github.com/uber/jaeger-client-go/config"
)

func main() {
    // Initialize Logrus, and set log level, output format and other parameters.
logrus. SetFormatter( & logrus. JSONFormatter{})
if isatty. IsTerminal(os. Stdout. Fd()) {
logrus. SetLevel(logrus. DebugLevel)
} else {
logrus. SetLevel(logrus. InfoLevel)
}
logrus. SetOutput(os. Stdout)

    // Initialize the Jaeger Tracer.
cfg, err := config.FromEnv()
if err != nil {
        log.Fatalf("Failed to read Jaeger config from env: %v", err)
    }
tracer, closer, err := cfg. NewTracer()
if err != nil {
log.Fatalf("Failed to create Jaeger tracer: %v", err)
}
defer closer. Close()
opentracing. SetGlobalTracer(tracer)

    //...
}
  1. Encapsulate Logrus and Jaeger for use in Golang gRPC-based microservice development:
import (
    "github.com/sirupsen/logrus"

    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/uber/jaeger-client-go"
    jaegercfg "github.com/uber/jaeger-client-go/config"
)

var log *logrus.Entry

func init() {
    log = logrus.WithFields(logrus.Fields{
        "app": "myapp",
        "env": "production",
    })
}

func doSomething(ctx context.Context) {
// Create a span.
span, ctx := opentracing. StartSpanFromContext(ctx, "doSomething")
defer span. Finish()

// Log.
log.WithFields(logrus.Fields{
"operation": span. OperationName(),
        "trace_id": span.Context().(jaeger.SpanContext).TraceID().String(),
        "span_id": span.Context().(jaeger.SpanContext).SpanID().String(),
        }).Infof("doing something")

// Initiate a downstream call to pass the current Span to the downstream service.
req := &pb.Request{...}
resp, err := client. Call(ctx, req)
if err != nil {...}

// Record the returned result and set the relevant label.
ext.HTTPStatusCode.Set(span, uint16(resp.StatusCode))
log.WithFields(logrus.Fields{
"operation": span. OperationName(),
        "trace_id": span.Context().(jaeger.SpanContext).TraceID().String(),
        "span_id": span.Context().(jaeger.SpanContext).SpanID().String(),
        "response_code": resp. StatusCode,
        }).Infof("got response")

//...
}

In this design, we use Logrus as the log component and Jaeger as the distributed link tracking component. Initialize Logrus and Jaeger in the main function, and set parameters such as log level and output format. When encapsulating Logrus and Jaeger, you can create a global log instance and tracer instance in the init function, and set some default field values, and then call the corresponding method where you need to print the log or record the Span. Note that the current context ctx must be passed to the relevant method when logging and recording Span, so as to transfer information across services. At the same time, when initiating a downstream call, the current span needs to be passed to the downstream service for link tracking.

Eight, redis data cache

In the development of microservices based on gRPC in Golang, in order to improve the performance and scalability of the system, it is often necessary to use caching technology to accelerate data access. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go

# Install the Redis client library.
$ go get github.com/go-redis/redis/v8
  1. Initialize RedisClient in the main function and set some parameters:
import (
    "github.com/go-redis/redis/v8"
)

var redisClient *redis.Client

func main() {
    // Initialize RedisClient.
    redisClient = redis.NewClient( &redis.Options{
        Addr: "localhost:6379",
        Password: "",
        DB: 0,
    })
}
  1. Encapsulate Redis cache related operations:
import (
    "time"

    "github.com/go-redis/redis/v8"
)

func getUserFromCache(userId string) (*User, error) {
// First read the user information from the cache.
key := fmt.Sprintf("user:%s", userId)
val, err := redisClient. Get(ctx, key). Result()
if err == nil {
user := &User{}
err := json. Unmarshal([]byte(val), user)
if err != nil {...}
return user, nil
}

if err != redis.Nil {...}

// If the user information does not exist in the cache, read it from the database and write it to the cache.
user, err := getUserFromDB(userId)
if err != nil {...}
val, err = json. Marshal(user)
if err != nil {...}
err = redisClient.Set(ctx, key, val, 1*time.Hour).Err()
if err != nil {...}

return user, nil
}

func updateUserCache(user *User) error {
// Update user information and write it to the cache.
key := fmt.Sprintf("user:%s", user.Id)
val, err := json. Marshal(user)
if err != nil {...}
err = redisClient.Set(ctx, key, val, 1*time.Hour).Err()
if err != nil {...}

return nil
}

func deleteUserFromCache(userId string) error {
// Delete the user information in the cache.
key := fmt.Sprintf("user:%s", userId)
err := redisClient.Del(ctx, key).Err()
if err != nil & amp; & amp; err != redis.Nil {...}

return nil
}

In this design, we use Redis as the data cache component and go-redis as the Redis client library. Initialize RedisClient in the main function and set some parameters. When encapsulating Redis cache-related operations, we first read data from the cache, if the data does not exist, read it from the database, and write it to the cache. During update and delete operations, it is also necessary to update or delete the corresponding data in the cache synchronously. Note that when performing Redis-related operations, the current context ctx needs to be passed for error handling and timeout control.

Nine, mysql data storage

In Golang’s gRPC-based microservice development, in order to persist data, the relational database MySQL is usually used. Here is a simple design example:

  1. Installation dependencies:
$ go get google.golang.org/grpc
$ go get github.com/golang/protobuf/proto
$ go get github.com/golang/protobuf/protoc-gen-go

# Install the MySQL client library.
$ go get github.com/go-sql-driver/mysql
  1. Initialize MySQLClient in the main function and set some parameters:
import (
    "database/sql"

    _ "github.com/go-sql-driver/mysql"
)

var mysqlDB *sql.DB

func main() {
    // Initialize MySQLClient.
    var err error
    mysqlDB, err = sql.Open("mysql", "user:password@tcp(localhost:3306)/dbname?charset=utf8mb4")
    if err != nil {...}
}
  1. Encapsulate MySQL storage related operations:
import (
"database/sql"
"errors"

_ "github.com/go-sql-driver/mysql"
)

type User struct {
Id string `json:"id"`
Name string `json:"name"`
}

func getUserFromDB(userId string) (*User, error) {
// Read user information from the database.
row := mysqlDB.QueryRow("SELECT id, name FROM users WHERE id=?", userId)
user := &User{}
err := row.Scan( &user.Id, &user.Name)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
return nil, err
}

return user, nil
}

func updateUserToDB(user *User) error {
// Update user information.
_, err := mysqlDB.Exec("UPDATE users SET name=? WHERE id=?", user.Name, user.Id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrNotFound
}
return err
}

return nil
}

func deleteUserFromDB(userId string) error {
// Delete user information in the database.
res, err := mysqlDB.Exec("DELETE FROM users WHERE id=?", userId)
if err != nil {...}
num, _ := res.RowsAffected()
if num == 0 {...}

return nil
}

In this design, we use MySQL as the data storage component and go-sql-driver/mysql as the MySQL client library. Initialize MySQLClient in the main function and set some parameters. When encapsulating MySQL storage-related operations, we can query, update, and delete operations through methods such as QueryRow and Exec. Note that error handling and transaction control are required for MySQL-related operations.