阶段性工作

.gitignore

@@ -0,0 +1,2 @@
+/.idea/
+/out/

build_linux_x64.bat

@@ -0,0 +1,9 @@
+go mod tidy
+
+@ECHO OFF
+SET CGO_ENABLED=0
+SET GOOS=linux
+SET GOARCH=amd64
+@ECHO ON
+go build -o out/ng-tail ./main.go
+

conf/ng-tail-test.conf

@@ -0,0 +1,10 @@
+{
+    "localPort": 8091,
+    "dbCfg": {
+        "dbAddr": "172.16.3.153",
+        "dbPort": 3306,
+        "dbSchema": "ticket-cloud",
+        "userName": "root",
+        "password": "123456"
+    }
+}

domain/po/nginx_access_log.go

@@ -0,0 +1,90 @@
+package po
+
+import (
+	"fmt"
+	"jollia.cn/golib/jokode"
+	"strings"
+	"time"
+)
+
+const tableNamePrefixNgAccLog = "ng_access_log"
+
+type NginxAccessLog struct {
+	Id        int64     `json:"id" gorm:"column:id;primary_key;AUTO_INCREMENT"`
+	RequestId string    `json:"request_id" gorm:"column:request_id;type:varchar(255);not null;comment:请求ID"`
+	TimeLocal time.Time `json:"time_local" gorm:"column:time_local;index;type:DATETIME;not null;comment:请求时间"`
+
+	Host       string `json:"host" gorm:"column:host;type:varchar(255);not null;comment:主机名"`
+	RequestStr string `json:"request" gorm:"-"`
+	Method     string `json:"-" gorm:"column:method;type:varchar(30);not null;comment:Http请求方法"`
+	Request    string `json:"-" gorm:"column:request;type:varchar(2000);index;not null;comment;请求 Url"`
+	HttpVer    string `json:"-" gorm:"column:http_ver;type:VARCHAR(10);not null;comment:Http请求版本"`
+	Status     int    `json:"status" gorm:"column:status;type:INT;not null;comment:响应状态码"`
+
+	RemoteAddr string `json:"remote_addr" gorm:"column:remote_addr;type:VARCHAR(255);not null;comment:客户端IP地址"`
+	Referer    string `json:"referer,omitempty" gorm:"column:referer;type:VARCHAR(255);comment:来源页面"`
+	Agent      string `json:"agent,omitempty" gorm:"column:agent;type:TEXT;comment:用户代理(浏览器信息)"`
+	XForwarded string `json:"x_forwarded,omitempty" gorm:"column:x_forwarded;type:VARCHAR(255);comment:X-Forwarded-For头域"`
+
+	RequestLengthStr string `json:"request_length,omitempty" gorm:"-"`
+	RequestLength    int64  `json:"-" gorm:"column:request_length;type:BIGINT;comment:请求字节数"`
+	BodyBytesSentStr string `json:"body_bytes_sent,omitempty" gorm:"-"`
+	BodyBytesSent    int64  `json:"-" gorm:"column:body_bytes_sent;type:BIGINT;comment:响应字节数"`
+
+	UpAddr         string  `json:"up_addr,omitempty" gorm:"column:up_addr;type:VARCHAR(255);comment:上游服务器地址"`
+	UpHost         string  `json:"up_host,omitempty" gorm:"column:up_host;type:VARCHAR(255);comment:上游服务器主机名"`
+	UpRespTimeStr  string  `json:"up_resp_time,omitempty" gorm:"-"`
+	UpRespTime     float64 `json:"-" gorm:"column:up_resp_time;type:FLOAT;comment:上游服务器响应时间"`
+	RequestTimeStr string  `json:"request_time,omitempty" gorm:"-"`
+	RequestTime    float64 `json:"-" gorm:"column:request_time;type:FLOAT;comment:响应时间"`
+}
+
+//func (l *NginxAccessLog) TableName() string {
+//	return tableNamePrefixNgAccLog
+//}
+
+func (l *NginxAccessLog) GetTableName() string {
+	return fmt.Sprintf("%s_%s", tableNamePrefixNgAccLog, l.TimeLocal.Format("20060102"))
+}
+
+func (l *NginxAccessLog) FixFields() {
+	if l.TimeLocal.IsZero() {
+		l.TimeLocal = time.Now()
+	}
+
+	l.Method, l.Request, l.HttpVer = splitRequest(l.RequestStr)
+
+	if l.RequestLengthStr != "" {
+		l.RequestLength = jokode.AtoI64(l.RequestLengthStr)
+	}
+
+	if l.BodyBytesSentStr != "" {
+		l.BodyBytesSent = jokode.AtoI64(l.BodyBytesSentStr)
+	}
+
+	if l.UpRespTimeStr != "" && l.UpRespTimeStr != "-" {
+		l.UpRespTime = jokode.AtoF64(l.UpRespTimeStr)
+	}
+
+	if l.RequestTimeStr != "" && l.RequestTimeStr != "-" {
+		l.RequestTime = jokode.AtoF64(l.RequestTimeStr)
+	}
+}
+
+func splitRequest(r string) (method, url, ver string) {
+	r = strings.TrimSpace(r)
+
+	if leftIdx := strings.Index(r, " "); leftIdx != -1 {
+		method = strings.TrimSpace(r[:leftIdx])
+
+		if rightIdx := strings.LastIndex(r, " "); rightIdx != -1 {
+			ver = r[rightIdx+1:]
+			url = r[leftIdx+1 : rightIdx]
+		} else {
+			url = r[leftIdx+1:]
+		}
+	} else {
+		url = r
+	}
+	return
+}

domain/po/nginx_access_log_test.go

@@ -0,0 +1,14 @@
+package po
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestNginxAccessLog_FixFields(t *testing.T) {
+	arg := &NginxAccessLog{}
+	line := `{"time_local": "2025-01-09T10:47:57+08:00", "remote_addr": "205.210.31.222", "referer": "-", "request": "\x16\x03\x01\x00\xEE\x01\x00\x00\xEA\x03\x03\x1C\xCD.\xCD\x04\xDF*\xD0\xFE\xD3&\xE5\x08.\x05\xFB\x80W\x129=\x91\x1F1\xECv\xE9\xF8D\xEA\xAA\xE8 \x92*}b\xBEA\x7F\x95I\x0B\x15\xEA\xB3\xBD\xD4\x8A\xC02o%wB\xA60Bf\xAAI\xCB\xAA\xE3|\x00&\xC0+\xC0/\xC0,\xC00\xCC\xA9\xCC\xA8\xC0\x09\xC0\x13\xC0", "status": 400, "body_bytes_sent": "173", "agent": "-", "x_forwarded": "-", "up_addr": "-","up_host": "-","up_resp_time": "-","request_time": "5.000", "request_length": "0", "request_id": "0fa54deb2fd2190b9f66b2f4b53cf66c", "host": "_" }`
+	if err := json.Unmarshal([]byte(line), arg); err != nil {
+		t.Errorf(err.Error())
+	}
+}

domain/queue.go

@@ -0,0 +1,40 @@
+package domain
+
+import (
+	"container/list"
+	"sync"
+)
+
+type QueueX struct {
+	locker  *sync.RWMutex
+	dataLst *list.List
+}
+
+func NewQueue() *QueueX {
+	return &QueueX{
+		locker:  new(sync.RWMutex),
+		dataLst: list.New(),
+	}
+}
+
+func (q *QueueX) Push(v interface{}) {
+	q.locker.Lock()
+	defer q.locker.Unlock()
+	q.dataLst.PushBack(v)
+}
+
+func (q *QueueX) Pop() interface{} {
+	q.locker.Lock()
+	defer q.locker.Unlock()
+	if e := q.dataLst.Front(); e != nil {
+		v := q.dataLst.Remove(e)
+		return v
+	}
+	return nil
+}
+
+func (q *QueueX) Empty() bool {
+	q.locker.RLock()
+	defer q.locker.RUnlock()
+	return q.dataLst.Len() == 0
+}

go.mod

@@ -0,0 +1,48 @@
+module nginx-tail
+
+go 1.23.4
+
+require (
+	github.com/gin-gonic/gin v1.10.0
+	github.com/pkg/errors v0.9.1
+	gorm.io/driver/mysql v1.5.7
+	gorm.io/gorm v1.25.12
+	jollia.cn/golib/jokode v0.8.7
+)
+
+require (
+	github.com/bwmarrin/snowflake v0.3.0 // indirect
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/natefinch/lumberjack v2.0.0+incompatible // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,122 @@
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
+github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
+golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+jollia.cn/golib/jokode v0.8.7 h1:3wQcvwkUnx+za8X+ibxX5QtdcvguasgeYqUyRy6laZ0=
+jollia.cn/golib/jokode v0.8.7/go.mod h1:Wyt+gccdr16dW3qPlYe0VuUkXt3r2zg525GoUl05qKM=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"jollia.cn/golib/jokode"
+	"nginx-tail/services"
+)
+
+const cfgFilePath = "conf/ng-tail.conf"
+
+func main() {
+	cfg := jokode.NewLogConfig()
+	cfg.FileName = "logs/ng-tail.log"
+	if err := jokode.InitLogger(cfg); err != nil {
+		panic(err)
+	}
+
+	jokode.Warnf("ng-tail starting ...")
+	defer jokode.Warnf("ng-tail stopping ...")
+
+	if err := services.LoadAppConfig(cfgFilePath); err != nil {
+		panic(err)
+	}
+
+	services.StartCliService()
+}

run_tail.sh

@@ -0,0 +1,2 @@
+chmod u+x ./ng-tail
+nohup ./ng-tail &

services/cli_servcie.go

@@ -0,0 +1,99 @@
+package services
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"nginx-tail/domain/po"
+	"os"
+)
+
+type DbConfig struct {
+	DbAddr   string `json:"dbAddr"`
+	DbPort   int    `json:"dbPort"`
+	DbSchema string `json:"dbSchema"`
+	UserName string `json:"userName"`
+	Password string `json:"password"`
+}
+
+type AppConfig struct {
+	LocalPort   int       `json:"localPort"`
+	LogFilePath string    `json:"logFilePath,omitempty"`
+	DbCfg       *DbConfig `json:"dbCfg"`
+}
+
+var (
+	appCfgInstance *AppConfig
+	dbInstance     *gorm.DB
+)
+
+func LoadAppConfig(cfgFilePath string) error {
+	if data, err := os.ReadFile(cfgFilePath); err != nil {
+		return errors.Wrapf(err, "try open config file '%s' fail", cfgFilePath)
+	} else {
+		appCfg := &AppConfig{}
+		if err = json.Unmarshal(data, appCfg); err != nil {
+			return errors.Wrapf(err, "try parse config file '%s' fail", cfgFilePath)
+		} else {
+			if appCfg.LogFilePath == "" {
+				appCfg.LogFilePath = "/var/log/nginx/access.log"
+			}
+
+			if err = initDb(appCfg.DbCfg); err != nil {
+				return errors.Wrapf(err, "try init db fail")
+			}
+			appCfgInstance = appCfg
+		}
+	}
+
+	return nil
+}
+
+func StartCliService() {
+	go TailFile(appCfgInstance.LogFilePath, 5)
+
+	g := gin.Default()
+	g.GET("/", func(c *gin.Context) {
+		c.JSON(200, gin.H{
+			"code": 0,
+			"msg":  "ok",
+		})
+	})
+
+	if err := g.Run(fmt.Sprintf(":%d", appCfgInstance.LocalPort)); err != nil {
+		panic(err)
+	}
+}
+
+func GetDb() *gorm.DB {
+	return dbInstance
+}
+
+func initDb(cfg *DbConfig) error {
+	if cfg == nil {
+		return errors.New("can not init db by nil config")
+	}
+
+	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local",
+		cfg.UserName, cfg.Password, cfg.DbAddr, cfg.DbPort, cfg.DbSchema)
+
+	if db, err := gorm.Open(mysql.Open(dsn),
+		&gorm.Config{Logger: logger.Default.LogMode(logger.Warn)}); err != nil {
+		return errors.Wrap(err, "try open db fail")
+	} else {
+		if err = autoMigrateObjects(db); err != nil {
+			return errors.Wrap(err, "try auto migrate objects fail")
+		}
+		dbInstance = db
+	}
+
+	return nil
+}
+
+func autoMigrateObjects(db *gorm.DB) error {
+	return db.AutoMigrate(&po.NginxAccessLog{})
+}

services/cli_servcie_test.go

@@ -0,0 +1,28 @@
+package services
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+)
+
+func TestLoadAppConfig(t *testing.T) {
+	cfg := &AppConfig{
+		LocalPort: 8091,
+		DbCfg: &DbConfig{
+			DbAddr:   "172.16.3.153",
+			DbPort:   3306,
+			DbSchema: "ticket-cloud",
+			UserName: "root",
+			Password: "123456",
+		},
+	}
+
+	if data, err := json.MarshalIndent(cfg, "", "    "); err != nil {
+		t.Fatal(err)
+	} else {
+		if err = os.WriteFile("ng-tail-test.conf", data, 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

services/tail_service.go

@@ -0,0 +1,165 @@
+package services
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"jollia.cn/golib/jokode"
+	"nginx-tail/domain"
+	"nginx-tail/domain/po"
+	"os"
+	"os/exec"
+	"sync"
+	"time"
+)
+
+var (
+	fileInstanceSize    int64
+	msgList             *domain.QueueX
+	lastTableName       string
+	lastTableNameLocker *sync.RWMutex
+)
+
+func init() {
+	msgList = domain.NewQueue()
+}
+
+func getLastTableName() string {
+	lastTableNameLocker.RLock()
+	defer lastTableNameLocker.RUnlock()
+
+	return lastTableName
+}
+
+func TailFile(filePath string, retryInterval time.Duration) {
+	var err error = nil
+
+	for {
+		err = tailProcessor(filePath)
+		if err != nil {
+			jokode.Errorf("tailProcessor error: %v", err)
+			return
+		}
+		time.Sleep(retryInterval * time.Second)
+	}
+}
+
+func testFileStat(filePath string, exitCh chan<- bool) {
+	for {
+		if fi, err := os.Stat(filePath); err != nil {
+			jokode.Warnf("try test file '%s' stat fail", filePath)
+			return
+		} else {
+			fs := fi.Size()
+			if fileInstanceSize > 0 && fs < fileInstanceSize {
+				// 可能发生了截断或者绕接 需要触发重新监听
+				exitCh <- true
+				break
+			}
+			fileInstanceSize = fs
+		}
+		time.Sleep(10 * time.Second)
+	}
+}
+
+func createTableDynamic(tableName string) {
+	var count int64
+	db := GetDb()
+
+	if err := db.Raw(fmt.Sprintf(`select count(1) from information_schema.tables where table_name='%s'`, tableName)).Scan(&count).Error; err != nil {
+		s := fmt.Sprintf("get dynamic table by name '%s' fail, error info: %v", tableName, err)
+		jokode.Error(s)
+		panic(s)
+	}
+
+	if count == 0 {
+		if err := db.Table(tableName).AutoMigrate(&po.NginxAccessLog{}); err != nil {
+			s := fmt.Sprintf("try create table '%s' fail, error info: %v", tableName, err)
+			jokode.Error(s)
+			panic(s)
+		}
+	}
+}
+
+func writeToDb() {
+	for {
+		db := GetDb()
+		if db == nil {
+			time.Sleep(2 * time.Second)
+		}
+
+		obj := msgList.Pop()
+		if obj == nil {
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+
+		if x, ok := obj.(*po.NginxAccessLog); ok {
+			x.FixFields()
+			if tableName := x.GetTableName(); tableName != getLastTableName() {
+				createTableDynamic(tableName)
+			}
+
+			if err := GetDb().Save(x).Error; err != nil {
+				jokode.Errorf("try save to db, error info: %v", err)
+			}
+		}
+
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+func tailProcessor(filePath string) error {
+	var stdout io.ReadCloser
+	defer func() {
+		if stdout != nil {
+			if err := stdout.Close(); err != nil {
+				jokode.Warnf("try close stdout pipe fail, error info: %v", err)
+			}
+		}
+	}()
+
+	chExit := make(chan bool, 1)
+
+	tail := exec.Command("tail", "-f", filePath)
+	jokode.Infof("try tail file '%s'", filePath)
+
+	if so, err := tail.StdoutPipe(); err != nil {
+		return errors.Wrap(err, "can not get std out pipe")
+	} else {
+		stdout = so
+	}
+
+	go testFileStat(filePath, chExit)
+	go writeToDb()
+
+	if err := tail.Start(); err != nil {
+		return errors.Wrap(err, "can not start tail")
+	}
+
+	reader := bufio.NewReader(stdout)
+	for {
+		select {
+		case <-chExit:
+			jokode.Warn("receive exit signal")
+			return nil
+		default:
+			if line, _, err := reader.ReadLine(); err != nil {
+				jokode.Errorf("try read stdout fail, error info: %v", err)
+				break
+			} else if len(line) == 0 {
+				// 没有读取到数据, 有可能 tail 进程已经推出
+				break
+			} else {
+				arg := &po.NginxAccessLog{}
+				if err = json.Unmarshal(line, arg); err != nil {
+					jokode.Errorf("try unmarshal '%s' fail, error info: %v", string(line), err)
+					continue
+				}
+				msgList.Push(arg)
+			}
+		}
+	}
+}

services/tail_service_test.go

@@ -0,0 +1,34 @@
+package services
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func Test(t *testing.T) {
+	str := "hello world this is a test"
+	x, y, z := splitString(str)
+	fmt.Println(x)
+	fmt.Println(y)
+	fmt.Println(z)
+}
+
+func splitString(str string) (beforeFirstSpace, afterLastSpace, middle string) {
+	firstSpaceIndex := strings.Index(str, " ")
+	if firstSpaceIndex != -1 {
+		beforeFirstSpace = str[:firstSpaceIndex]
+
+		lastSpaceIndex := strings.LastIndex(str, " ")
+		if lastSpaceIndex != -1 {
+			afterLastSpace = str[lastSpaceIndex+1:]
+			middle = str[firstSpaceIndex+1 : lastSpaceIndex]
+		}
+	}
+	return
+}
+
+func TestNewDate(t *testing.T) {
+	str := "07/Jan/2025:12:42:46 +0800"
+	fmt.Println(str)
+}