diff --git a/go.mod b/go.mod index aa72c9aa72087ce25ab7c622e014459d2e53edf9..ff368fd07cbac8964d253cac427fc77024ad10e8 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/agrison/go-tablib v0.0.0-20160310143025-4930582c22ee // indirect github.com/agrison/mxj v0.0.0-20160310142625-1269f8afb3b4 // indirect github.com/bndr/gotabulate v1.1.2 // indirect + github.com/clbanning/mxj v1.8.4 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/gin-gonic/contrib v0.0.0-20190923054218-35076c1b2bea github.com/gin-gonic/gin v1.4.0 diff --git a/go.sum b/go.sum index a4d60bc13ce129ed4970005a45d41d482b878da6..2f20c974298124668d3985459525386d03f48490 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= +gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Chronokeeper/anyxml v0.0.0-20160530174208-54457d8e98c6 h1:Etfj2lhXyrYemgmWzEtEQQb1kezeEXc8jVjkQUyJnWI= @@ -21,6 +22,8 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bndr/gotabulate v1.1.2 h1:yC9izuZEphojb9r+KYL4W9IJKO/ceIO8HDwxMA24U4c= github.com/bndr/gotabulate v1.1.2/go.mod h1:0+8yUgaPTtLRTjf49E8oju7ojpU11YmXyvq1LbPAb3U= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -104,6 +107,7 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -141,6 +145,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -198,6 +203,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -217,6 +223,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/flosch/pongo2.v3 v3.0.0-20141028000813-5e81b817a0c4 h1:eyQQg/uGuZ3ndaBhqteakHpVW+dSOPalilfC9RpM2TA= gopkg.in/flosch/pongo2.v3 v3.0.0-20141028000813-5e81b817a0c4/go.mod h1:bJkYqV5pg6+Z7MsSu/hSb1zsPT955hBW2QHLE1jm4wA= diff --git a/src/handler/base.go b/src/handler/base.go new file mode 100644 index 0000000000000000000000000000000000000000..d0fe9c7905033f77c67769e8d2444d1f86ed1889 --- /dev/null +++ b/src/handler/base.go @@ -0,0 +1,16 @@ +package handler + +import ( + "github.com/gin-gonic/gin" + "gitlab.wodcloud.com/apaas/apaas-meshproxy/src/tools" + "time" +) + +func Health(c *gin.Context) { + c.JSON(200, gin.H{ + "host": c.Request.Host, + "header": c.Request.Header, + "serverTime": time.Now(), + "ip": tools.RemoteIp(c.Request), + }) +} diff --git a/src/handler/proxyhandler.go b/src/handler/proxyhandler.go index ae3a48258bfa76e26c14bfe780e3d45567d1108b..b4dc0f6d3920d463bb3498ba005e34e10fa843ee 100644 --- a/src/handler/proxyhandler.go +++ b/src/handler/proxyhandler.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/vulcand/oxy/forward" "github.com/vulcand/oxy/testutils" + "gitlab.wodcloud.com/apaas/apaas-meshproxy/src/model" "gitlab.wodcloud.com/apaas/apaas-meshproxy/src/service" "io/ioutil" "net/http" @@ -16,19 +17,24 @@ import ( var Resp *http.Response func Proxy(c *gin.Context) { - appId := c.Param("appId") + applyId := c.Query("applyId") + if applyId == "" { + res := model.WebRes{} + res.Data = "applyId不能为空" + c.JSON(200, res) + return + } //获取服务相关信息 - proxyData, err := service.GetRealPath(appId) + proxyData, err := service.GetRealPath(applyId) if err != nil { c.Error(err) return } - f, _ := forward.New(forward.PassHostHeader(true), forward.ResponseModifier(func(resp *http.Response) error { Resp = resp - //判断是否可以调用 - callflag, sensitiveflag, err := service.QueryCallsCount(appId, proxyData) + //判断是否超过调用次数是否可以调用 + callflag, sensitiveflag, err := service.QueryCallsCount(proxyData) if err != nil { Return(err.Error()) return nil @@ -37,11 +43,8 @@ func Proxy(c *gin.Context) { Return(`调用达到当日限定次数`) return nil } - respbody, _ := ioutil.ReadAll(resp.Body) - var res interface{} - res = respbody //是否需要过滤字段 if proxyData.ResFields != "" { @@ -52,7 +55,9 @@ func Proxy(c *gin.Context) { res = service.SensitiveFilter(proxyData, res) } //调用痕迹进入队列 - service.RecordCall(proxyData, resp.StatusCode, res) + //service.RecordCall(proxyData, resp.StatusCode, respbody) + //服务调用计次 + service.CallCounts(proxyData) Return(res) return nil })) diff --git a/src/main.go b/src/main.go index 7f7a2afdd2ffb5c26c7e4fdfbd5bbd14b7015ea3..35e47e031414b7e733caccccf3da44157716f47a 100644 --- a/src/main.go +++ b/src/main.go @@ -18,10 +18,10 @@ import ( var ( argPort = pflag.Int("port", 8011, "") argPrefix = pflag.String("prefix", "/bgmesh/fiddler", "") - redisUrl = pflag.String("redisUrl", "redis://localhost:63793", "") + redisUrl = pflag.String("redisUrl", "redis://localhost:63799", "") redisTag = pflag.String("redisTag", "apaas-mesh-proxy", "") confPath = pflag.String("confPath", "/app/config/proxy.json", "") - dbURL = pflag.String("dbURL", "host=localhost port=5432 user=postgres password=passwd123 dbname=apaas sslmode=disable", "") + dbURL = pflag.String("dbURL", "host=localhost port=54323 user=postgres password=passwd123 dbname=apaas sslmode=disable", "") dbDriverName = pflag.String("dbDriverName", "postgres", "") ) diff --git a/src/model/response.go b/src/model/response.go index 7897d65fe9de4f73684d1259dd99e958b968c4e3..d4b0ef7ad4d5a02226f9ee0ba20c5a0bac3b6f12 100644 --- a/src/model/response.go +++ b/src/model/response.go @@ -10,3 +10,9 @@ type ProxyData struct { Service_id int `json:"service_id" xorm:"service_id"` User_id string `json:"user_id" xorm:"user_id"` } + +type WebRes struct { + Success int `json:"success"` + ErrMsg string `json:"errMsg,omitempty"` + Data interface{} `json:"data,omitempty"` +} diff --git a/src/router/router.go b/src/router/router.go index 603f5c7883c783a8931522bc15455e3934ba7526..29756792a15b11aebc2ce56ca848fd7ead97a35c 100644 --- a/src/router/router.go +++ b/src/router/router.go @@ -33,5 +33,6 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { { root.Any("/:appId/proxy", handler.Proxy) } + e.GET("/health", handler.Health) return e } diff --git a/src/service/field.go b/src/service/field.go index f2ecf950a8159ff1cb3c75ec82bd94e308ccad6a..96bce2eba8a6e12eea886cea609b925b769d080f 100644 --- a/src/service/field.go +++ b/src/service/field.go @@ -26,27 +26,24 @@ func CheckSensitiveField() bool { } //获取当日调用次数 返回是否到达当日访问量，是否到达敏感字段访问量 -func QueryCallsCount(applyId string, filter model.ProxyData) (call bool, sensitiveCall bool, err error) { +func QueryCallsCount(filter model.ProxyData) (call bool, sensitiveCall bool, err error) { //连接redis redis, err := client.GetRedisClient() if err != nil { return } - countStr, err := redis.Get(applyId) + date := time.Now().Format(config.LocalDateFormat) + key := fmt.Sprintf("%s-%s", date, filter.Apply_id) + countStr, err := redis.Get(key) if err != nil && err.Error() != `redis: nil` { return } - //// 更新调用次数 - //err = UpdateCallsCount(applyId) - //if err != nil { - // return false, err - //} var count int call = true sensitiveCall = true if countStr == "" { expire := GetExpire() - err = redis.Set(applyId, 1, expire) + err = redis.Set(key, 1, expire) count = 1 } else { count, err = strconv.Atoi(countStr) @@ -62,7 +59,7 @@ func QueryCallsCount(applyId string, filter model.ProxyData) (call bool, sensiti sensitiveCall = false } expire := GetExpire() - err = redis.Set(applyId, count+1, expire) + err = redis.Set(key, count+1, expire) } return } @@ -81,7 +78,7 @@ func GetExpire() time.Duration { } //记录调用痕迹 -func RecordCall(proxyData model.ProxyData, status int, res interface{}) { +func RecordCall(proxyData model.ProxyData, status int, res []byte) { acc := dao.Service_request_record{} redis, err := client.GetRedisClient() if err != nil { @@ -89,7 +86,10 @@ func RecordCall(proxyData model.ProxyData, status int, res interface{}) { return } if status != 200 { - acc.Error = string(res.([]byte)) + acc.Error = string(res[:]) + //mjson,_ :=json.Marshal(res) + //mString :=string(mjson) + //acc.Error = mString } acc.Request_time = tools.JsonTime(time.Now()) acc.Apply_id = proxyData.Apply_id @@ -97,11 +97,8 @@ func RecordCall(proxyData model.ProxyData, status int, res interface{}) { acc.Service_id = proxyData.Service_id acc.Date_day = time.Now().Day() fmt.Println(time.Now().Month().String()) - //acc.Date_month,err = strconv.Atoi(time.Now().Month().String()) - //if err!=nil{ - // fmt.Println(err.Error()) - // return - //} + acc.Date_month = int(time.Now().Month()) + acc.Date_year = time.Now().Year() acc.Status = status b, _ := json.Marshal(acc) @@ -114,6 +111,47 @@ func RecordCall(proxyData model.ProxyData, status int, res interface{}) { return } +//服务调用计次 +func CallCounts(proxyData model.ProxyData) (err error) { + redis, err := client.GetRedisClient() + if err != nil { + fmt.Println(err.Error()) + return + } + date := time.Now().Format(config.LocalDateFormat) + aMonBefore := time.Now().AddDate(0, -1, 0).Format(config.LocalDateFormat) + + str, err := redis.Get(string(proxyData.Service_id)) + if err != nil && err.Error() != `redis: nil` { + return + } + if err != nil { + return + } + dateMap := make(map[string]int) + if str != "" { + err = json.Unmarshal([]byte(str), dateMap) + if err != nil { + return + } + if _, ok := dateMap[date]; ok { + dateMap[date] = dateMap[date] + 1 + } else { + dateMap[date] = 1 + } + for k, _ := range dateMap { + if tools.CompareDate(k, aMonBefore) { + delete(dateMap, k) + } + } + + } else { + dateMap[date] = 1 + } + err = redis.Set(string(proxyData.Service_id), dateMap, -1) + return +} + //过滤返回字段 func FiledFilter(proxyData model.ProxyData, respbody []byte) interface{} { model := make(map[string]interface{}) @@ -280,7 +318,7 @@ func FilterSensituveField(model interface{}, returnData interface{}) (body inter } // 获取真实地址和返回结构、调用限定次数 -func GetRealPath(appId string) (res model.ProxyData, err error) { +func GetRealPath(applyId string) (res model.ProxyData, err error) { db, err := client.GetConnect() if err != nil { return @@ -289,12 +327,12 @@ func GetRealPath(appId string) (res model.ProxyData, err error) { Select(`service_apply.res_fields as res_fields,service.req_url as req_url,service_request_spcs.count as count,service_safe_config.sensituve_word as sensituve_word ,service_request_spcs.sensitive_count as sensitive_count,service_apply.service_id as service_id,service_apply.id as apply_id,service_apply.user_id as user_id`). Table(`service`). - Join(`left`, `service_apply`, `service_apply.service_id = service.id`). + Join(`inner`, `service_apply`, `service_apply.service_id = service.id`). Join(`left`, `service_request_spcs`, `service_request_spcs.id = service_apply.request_spcs_id`). Join(`left`, `service_safe_config`, `service_apply.service_id = service_safe_config.service_id`). - Where(`service_apply.id=?`, appId).Get(&res) + Where(`service_apply.id=?`, applyId).Get(&res) if !has { - err = errors.New(`未找到申请服务！`) + err = errors.New(`未找到发布的服务！`) } return } diff --git a/src/tools/tool.go b/src/tools/tool.go index 43d961a3de372536765443e457b062914a101799..f858e44e7384b1758ce5a3527cd286e1f08a96f9 100644 --- a/src/tools/tool.go +++ b/src/tools/tool.go @@ -22,3 +22,13 @@ type JsonDate time.Time func (j JsonDate) MarshalJSON() ([]byte, error) { return []byte(`"` + time.Time(j).Format(config.LocalDateFormat) + `"`), nil } + +//date1 小于于 date2 返回 +func CompareDate(date1, date2 string) bool { + t1time, _ := time.ParseInLocation("2006-01-02 15:04:05", date1, time.Local) + t2time, _ := time.ParseInLocation("2006-01-02 15:04:05", date2, time.Local) + if t1time.Before(t2time) { + return true + } + return false +}