服务调用（未实计次入库）

.beagle.yml

 workspace:
   path: src/gitlab.wodcloud.com/apaas/apaas-meshproxy
-
+platform: 10.11.92.33
 clone:
   git:
     image: registry.cn-qingdao.aliyuncs.com/wod-devops/git:1.5.0
-
+    dns: 223.5.5.5
 pipeline:
   go:
     image: registry.cn-qingdao.aliyuncs.com/wod-devops/go:1.13.1-stretch

go.mod

@@ -3,13 +3,28 @@ module gitlab.wodcloud.com/apaas/apaas-meshproxy
 go 1.12
 
 require (
+	github.com/Chronokeeper/anyxml v0.0.0-20160530174208-54457d8e98c6 // indirect
+	github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
+	github.com/CloudyKit/jet v2.1.2+incompatible // indirect
+	github.com/agrison/go-tablib v0.0.0-20160310143025-4930582c22ee // indirect
+	github.com/agrison/mxj v0.0.0-20160310142625-1269f8afb3b4 // indirect
+	github.com/bndr/gotabulate v1.1.2 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
 	github.com/gin-gonic/contrib v0.0.0-20190923054218-35076c1b2bea
 	github.com/gin-gonic/gin v1.4.0
 	github.com/go-redis/redis v6.15.6+incompatible
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/lib/pq v1.4.0
 	github.com/mattn/go-isatty v0.0.8 // indirect
 	github.com/onsi/ginkgo v1.10.2 // indirect
 	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/pflag v1.0.5
+	github.com/tealeg/xlsx v1.0.5 // indirect
 	github.com/vulcand/oxy v1.0.0
+	github.com/xormplus/builder v0.0.0-20200331055651-240ff40009be // indirect
+	github.com/xormplus/core v0.0.0-20200308074340-f3bce19d5f31
+	github.com/xormplus/xorm v0.0.0-20200410045938-f6b4c1cd3b8b
+	gopkg.in/flosch/pongo2.v3 v3.0.0-20141028000813-5e81b817a0c4 // indirect
+	xorm.io/cmd v0.0.0-20200427014343-9ccdb0ebedf7 // indirect
 )

src/client/db.go

+package client
+
+import (
+	"fmt"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+	"github.com/sirupsen/logrus"
+	"github.com/xormplus/core"
+	"github.com/xormplus/xorm"
+	"gitlab.wodcloud.com/apaas/apaas-meshproxy/src/config"
+	"time"
+)
+
+var DB *xorm.Engine
+
+func GetConnect() (xormDB *xorm.Engine, err error) {
+	//if DB != nil && DB.Ping() == nil {
+	if DB != nil {
+		return DB, nil
+	}
+	xormDB, err = xorm.NewEngine(config.DriverName, config.DbURL)
+	if err != nil {
+		logrus.Errorf("error creating db instance to %s: %s", config.DbURL, err)
+		return nil, err
+	}
+	if err = xormDB.Ping(); err != nil {
+		logrus.Errorf("error creating db connection to %s: %s", config.DbURL, err)
+		return nil, err
+	}
+	xormDB.SetMapper(core.SnakeMapper{})
+	xormDB.ShowSQL(true)
+	DB = xormDB
+	DB.TZLocation, _ = time.LoadLocation("Asia/Shanghai")
+	fmt.Println("connect Db success!")
+	return xormDB, err
+}

src/config/conf.go

 package config
 
 var (
-	RedisURL string
-	RedisTag string
-	Prefix   string
+	RedisURL   string
+	RedisTag   string
+	Prefix     string
+	Url        string
+	RedisUrl   string
+	DriverName string
+	DbURL      string
+	EsHost     []string
 )
 
 var ProxyConf = struct {
@@ -17,3 +22,8 @@ type Param struct {
 	Name  string `json:"name"`
 	Value string `json:"value"`
 }
+
+const LocalDateTimeFormat string = "2006-01-02 15:04:05"
+const LocalDateTimeFormatStr string = "20060102150405"
+const LocalTimeFormat string = "15:04"
+const LocalDateFormat string = "2006-01-02"

src/dao/service.go

+package dao
+
+type Service struct {
+	Id              int    `json:"id" xorm:"id pk"`   // 主键
+	Name            string `json:"name"`              // 名称
+	Domain          string `json:"domain"`            // 领域
+	Organization    int    `json:"organization"`      // 组织
+	Cover           string `json:"cover"`             // 封面
+	Openness        int    `json:"openness"`          // 开放程度
+	RealUrl         string `json:"real_url"`          // 真实地址
+	ProxyUrl        string `json:"proxy_url"`         // 代理地址
+	State           int    `json:"state"`             // 服务状态 0 下架 1 上架
+	UserId          int    `json:"user_id"`           // 用户id
+	CreateTime      string `json:"create_time"`       // 创建时间
+	DownTime        string `json:"down_time"`         // 下架时间
+	DataServiceType int    `json:"data_service_type"` // 1 原始服务  2 专题服务
+}
+
+func (Service) TableName() string {
+	return `apaasv3dev.service`
+}

src/dao/service_apply.go

+package dao
+
+type ServiceApply struct {
+	Id            int    `json:"id" xorm:"id"`
+	Service_id    string `json:"service_id" xorm:"service_id"`
+	User_id       int    `json:"user_id" xorm:"user_id"`
+	Request_count int    `json:"request_count" xorm:"request_count"`
+	ResFields     string `json:"res_fields" xorm:"res_fields text"`
+}
+
+func (ServiceApply) TableName() string {
+	return `apaasv3dev.service_apply`
+}

src/dao/service_req_field.go

+package dao

src/dao/service_spcs.go

+package dao

src/handler/proxyhandler.go

@@ -2,6 +2,7 @@ package handler
 
 import (
 	"bytes"
+	"encoding/json"
 	"github.com/gin-gonic/gin"
 	"github.com/vulcand/oxy/forward"
 	"github.com/vulcand/oxy/testutils"
@@ -12,38 +13,65 @@ import (
 	"strings"
 )
 
+var Resp *http.Response
+
 func Proxy(c *gin.Context) {
-	appId := c.Query("appId")
+	appId := c.Param("appId")
 	// TODO 根据appId 获取真实地址
-	realPath, err := service.GetRealPath(appId)
+	proxyData, err := service.GetRealPath(appId)
 	if err != nil {
 		c.Error(err)
 		return
 	}
 	f, _ := forward.New(forward.PassHostHeader(true), forward.ResponseModifier(func(resp *http.Response) error {
-		//defer resp.Body.Close()
-		// TODO 是否需要处理数据  ( 是否设置了敏感字段 )
-		if c.Request.Method == "GET" && service.CheckSensitiveField() {
-			// TODO 过滤敏感字段
-			respbody, _ := ioutil.ReadAll(resp.Body)
-			respbodyNew, err := service.FilterSensituveField(respbody)
-			if err != nil {
-				return err
-			}
-			resp.Header.Set("X-Log-By", "Apaas")
-			l := strconv.Itoa(len(respbodyNew))
-			resp.Header.Set("Content-Length", l)
-			resp.Body = ioutil.NopCloser(bytes.NewBuffer(respbodyNew))
+		Resp = resp
+		//  判断调用是否到达上限
+		callflag, sensitiveflag, err := service.QueryCallsCount(appId, proxyData)
+		if callflag == false {
+			Return(`调用达到当日限定次数`)
+			return nil
+		}
+
+		respbody, _ := ioutil.ReadAll(resp.Body)
+		//筛选字段
+		model := make(map[string]interface{})
+		realData := make(map[string]interface{})
+		json.Unmarshal([]byte(proxyData.ResFields), &model)
+		json.Unmarshal(respbody, &realData)
+		res := service.Change(model, realData)
+		if err != nil {
+			Return(err.Error())
+			return nil
+		}
+		// TODO 筛选过滤敏感字段
+		if sensitiveflag == false {
+			Sensituve_word := make(map[string]interface{})
+			json.Unmarshal([]byte(proxyData.Sensituve_word), &Sensituve_word)
+			res = service.FilterSensituveField(Sensituve_word, res)
 		}
-		// TODO 计次计费
+		Return(res)
 		return nil
 	}))
-	c.Request.URL = testutils.ParseURI(realPath)
-	c.Request.RequestURI = realPath
-	c.Request.Host = getHost(realPath)
+	c.Request.URL = testutils.ParseURI(proxyData.RealUrl)
+	c.Request.RequestURI = proxyData.RealUrl
+	c.Request.Host = getHost(proxyData.RealUrl)
 	f.ServeHTTP(c.Writer, c.Request)
 }
 
+func Return(res interface{}) {
+	var b []byte
+	switch res.(type) {
+	case []byte:
+		b = res.([]byte)
+	case interface{}:
+		b, _ = json.Marshal(res)
+	}
+	Resp.Header.Set("X-Log-By", "Apaas")
+	l := strconv.Itoa(len(b))
+	Resp.Header.Set("Content-Length", l)
+	Resp.Body = ioutil.NopCloser(bytes.NewBuffer(b))
+}
+
 // 获取域名
 func getHost(url string) (path string) {
 	if strings.Contains(url, "//") {

src/main.go

@@ -16,11 +16,13 @@ import (
 )
 
 var (
-	argPort   = pflag.Int("port", 8088, "")
-	argPrefix = pflag.String("prefix", "/bgmesh/fiddler", "")
-	redisUrl  = pflag.String("redisUrl", "redis://k8s.wodcloud.com:16379", "")
-	redisTag  = pflag.String("redisTag", "apaas-mesh-proxy", "")
-	confPath  = pflag.String("confPath", "/app/config/proxy.json", "")
+	argPort      = pflag.Int("port", 8088, "")
+	argPrefix    = pflag.String("prefix", "/bgmesh/fiddler", "")
+	redisUrl     = pflag.String("redisUrl", "redis://localhost:6333", "")
+	redisTag     = pflag.String("redisTag", "apaas-mesh-proxy", "")
+	confPath     = pflag.String("confPath", "/app/config/proxy.json", "")
+	dbURL        = pflag.String("dbURL", "host=localhost port=5432 user=postgres password=passwd123 dbname=apaas sslmode=disable", "")
+	dbDriverName = pflag.String("dbDriverName", "postgres", "")
 )
 
 func main() {
@@ -52,4 +54,6 @@ func initConfig() {
 	config.RedisTag = *redisTag
 	config.RedisURL = *redisUrl
 	config.Prefix = *argPrefix
+	config.DriverName = *dbDriverName
+	config.DbURL = *dbURL
 }

src/model/response.go

+package model
+
+type ProxyData struct {
+	RealUrl         string `json:"req_url" xorm:"req_url"`
+	ResFields       string `json:"res_fields" xorm:"res_fields text"`
+	Count           int    `json:"count" xorm:"count"`
+	Sensituve_word  string `json:"sensituve_word" xorm:"sensituve_word"`
+	Sensitive_count int    `json:"sensitive_count" xorm:"sensitive_count"`
+}

src/router/router.go

@@ -31,7 +31,7 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
 	e.Use(middleware...)
 	root := e.Group(config.Prefix)
 	{
-		root.Any("/proxy", handler.Proxy)
+		root.Any("/:appId/proxy", handler.Proxy)
 	}
 	return e
 }

src/service/field.go

@@ -8,7 +8,12 @@
 package service
 
 import (
+	"errors"
 	"fmt"
+	"gitlab.wodcloud.com/apaas/apaas-meshproxy/src/client"
+	"gitlab.wodcloud.com/apaas/apaas-meshproxy/src/model"
+	"strconv"
+	"time"
 )
 
 // 判断是否设置了敏感字段，并有敏感词
@@ -16,15 +21,217 @@ func CheckSensitiveField() bool {
 	return true
 }
 
-// 过滤敏感词
-func FilterSensituveField(respbody []byte) (body []byte, err error) {
-	body = respbody
-	fmt.Println("=========> 处理返回体!")
-	return body, err
+//获取当日调用次数  返回是否到达当日访问量，是否到达敏感字段访问量
+func QueryCallsCount(applyId string, filter model.ProxyData) (call bool, sensitiveCall bool, err error) {
+	//连接redis
+	redis, err := client.GetRedisClient()
+	if err != nil {
+		return
+	}
+	countStr, err := redis.Get(applyId)
+	if err != nil && err.Error() != `redis: nil` {
+		return
+	}
+	////  更新调用次数
+	//err = UpdateCallsCount(applyId)
+	//if err != nil {
+	//	return false, err
+	//}
+	var count int
+	call = true
+	sensitiveCall = true
+	if countStr == "" {
+		expire := GetExpire()
+		err = redis.Set(applyId, 1, expire)
+		count = 1
+	} else {
+		count, err = strconv.Atoi(countStr)
+		if err != nil {
+			return
+		}
+		if count+1 > filter.Count {
+			call = false
+			return
+		}
+		//超过允许访问敏感字段次数
+		if count+1 > filter.Sensitive_count {
+			sensitiveCall = false
+		}
+		expire := GetExpire()
+		err = redis.Set(applyId, count+1, expire)
+	}
+	return
+}
+
+//获取过期时间
+func GetExpire() time.Duration {
+	now := time.Now()
+	timeStr := time.Now().AddDate(0, 0, 1).Format("2006-01-02")
+	fmt.Println("timeStr:", timeStr)
+	zero, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := zero
+	fmt.Println("timeNumber:", timeNumber)
+	subMin := now.Sub(zero)
+	fmt.Println(subMin.Minutes(), "分钟")
+	return subMin
+}
+
+//返回申请字段
+func Change(model interface{}, returnData interface{}) interface{} {
+	switch t := model.(type) {
+	case map[string]interface{}:
+		// TODO 判定值是否相等
+		for k, v := range returnData.(map[string]interface{}) {
+			flag := false
+			for k1, v1 := range model.(map[string]interface{}) {
+				if k == k1 {
+					flag = true
+					Change(v, v1)
+				}
+			}
+			// 删除属性
+			if !flag {
+				fmt.Println("删除属性：", k)
+				delete(returnData.(map[string]interface{}), k)
+			}
+		}
+		break
+	case []interface{}:
+		for k, v := range returnData.([]interface{}) {
+			switch v.(type) {
+			case map[string]interface{}:
+				for _, a2 := range model.([]interface{}) {
+					for k1, v1 := range v.(map[string]interface{}) {
+						flag := false
+						for k2, v2 := range a2.(map[string]interface{}) {
+							if k1 == k2 {
+								flag = true
+								Change(v1, v2)
+							}
+						}
+						// 删除属性
+						if !flag {
+							fmt.Println("删除属性：", k)
+							delete(v.(map[string]interface{}), k1)
+						}
+					}
+				}
+				break
+			case []interface{}:
+				for _, v2 := range model.([]interface{}) {
+					fmt.Println(v2)
+					Change(v2, v)
+				}
+				break
+			default:
+				// TODO 判断old是否有这个值
+				fmt.Println("数组类型 |||||||||||||||||||||||||||||||| old=====new=====>: ", model, returnData)
+			}
+		}
+		break
+	default:
+		fmt.Println("old=====new=====>: ", model, returnData)
+		fmt.Println("类型是*******>：", t)
+	}
+	return returnData
+}
+
+//更新调用次数
+func UpdateCallsCount(applyId string) (err error) {
+	db, err := client.GetConnect()
+	if err != nil {
+		return
+	}
+	sql := `update service_apply set request_count = request_count+1  where service_apply.service_id = '?'`
+	db.SQL(sql, applyId)
+
+	return
+}
+
+// 过滤敏感字段
+func FilterSensituveField(model interface{}, returnData interface{}) (body interface{}) {
+	switch t := model.(type) {
+	case map[string]interface{}:
+		// TODO 判定值是否相等
+		for k, v := range returnData.(map[string]interface{}) {
+			//flag := false
+			for k1, v1 := range model.(map[string]interface{}) {
+				if k == k1 {
+					//flag = true
+					switch v.(type) {
+					case string:
+						returnData.(map[string]interface{})[k] = `xxx`
+					}
+					FilterSensituveField(v, v1)
+				}
+			}
+			// 删除属性
+			//if !flag {
+			//	fmt.Println("删除属性：", k)
+			//	delete(returnData.(map[string]interface{}), k)
+			//}
+		}
+		break
+	case []interface{}:
+		for _, v := range returnData.([]interface{}) {
+			switch v.(type) {
+			case map[string]interface{}:
+				for _, a2 := range model.([]interface{}) {
+					for k1, v1 := range v.(map[string]interface{}) {
+						//flag := false
+						for k2, v2 := range a2.(map[string]interface{}) {
+							if k1 == k2 {
+								//flag = true
+								switch v.(map[string]interface{})[k1].(type) {
+								case string:
+									v.(map[string]interface{})[k1] = `xxx`
+								}
+								FilterSensituveField(v1, v2)
+							}
+						}
+						// 删除属性
+						//if !flag {
+						//	fmt.Println("删除属性：", k)
+						//	delete(v.(map[string]interface{}), k1)
+						//}
+					}
+				}
+				break
+			case []interface{}:
+				for _, v2 := range model.([]interface{}) {
+					fmt.Println(v2)
+					FilterSensituveField(v2, v)
+				}
+				break
+			default:
+				// TODO 判断old是否有这个值
+				fmt.Println("数组类型 |||||||||||||||||||||||||||||||| old=====new=====>: ", model, returnData)
+			}
+		}
+		break
+	default:
+		fmt.Println("old=====new=====>: ", model, returnData)
+		fmt.Println("类型是*******>：", t)
+	}
+	return returnData
 }
 
-// 获取真实地址
-func GetRealPath(appId string) (path string, err error) {
-	path = "https://www.baidu.com"
+// 获取真实地址和返回结构、调用限定次数
+func GetRealPath(appId string) (res model.ProxyData, err error) {
+	db, err := client.GetConnect()
+	if err != nil {
+		return
+	}
+	has, err := db.
+		Select(`service_apply.res_fields as res_fields,service.req_url as req_url,service_request_spcs.count as count,service_safe_config.sensituve_word as sensituve_word
+,service_request_spcs.sensitive_count as sensitive_count`).
+		Table(`apaasv3dev.service`).
+		Join(`left`, `apaasv3dev.service_apply`, `service_apply.service_id = service.id`).
+		Join(`left`, `apaasv3dev.service_request_spcs`, `service_request_spcs.id = service_apply.request_spcs_id`).
+		Join(`left`, `apaasv3dev.service_safe_config`, `service_apply.service_id = service_safe_config.service_id`).
+		Where(`service_apply.id=?`, appId).Get(&res)
+	if !has {
+		err = errors.New(`未找到申请服务！`)
+	}
 	return
 }

src/tools/tool.go

+package tools
+
+import (
+	"gitlab.wodcloud.com/apaas/apaas-meshproxy/src/config"
+	"time"
+)
+
+type JsonTime time.Time
+
+func (j JsonTime) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + time.Time(j).Format(config.LocalDateTimeFormat) + `"`), nil
+}
+
+type JsonHour time.Time
+
+func (j JsonHour) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + time.Time(j).Format(config.LocalTimeFormat) + `"`), nil
+}
+
+type JsonDate time.Time
+
+func (j JsonDate) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + time.Time(j).Format(config.LocalDateFormat) + `"`), nil
+}
+
+type JsonTime time.Time
+
+func (j JsonTime) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + time.Time(j).Format(conf.LocalDateTimeFormat) + `"`), nil
+}