binlog-parser/main.go

package main

import (
	"b612.me/mysql/binlog"
	"b612.me/mysql/gtid"
	"b612.me/starlog"
	"b612.me/staros"
	"fmt"
	"github.com/spf13/cobra"
	"github.com/tealeg/xlsx"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

var (
	bigThan     int
	smallThan   int
	startPos    int
	endPos      int
	startTime   int64
	endTime     int64
	includeGtid string
	excludeGtid string
	filePath    []string
	outPath     string
	vbo         bool
	skipquery   bool
	pos         int64
	prefix      string
)

func init() {
	cmd.Flags().IntVarP(&endPos, "pos", "P", 0, "skipPos of binlog")
	cmd.Flags().IntVarP(&startPos, "start-pos", "S", 0, "startPos of binlog")
	cmd.Flags().IntVarP(&endPos, "end-pos", "E", 0, "endPos of binlog")
	cmd.Flags().StringVarP(&includeGtid, "include-gtid", "i", "", "include gtid")
	cmd.Flags().StringVarP(&excludeGtid, "exclude-gtid", "e", "", "exclude gtid")
	cmd.Flags().StringSliceVarP(&filePath, "path", "p", []string{}, "binlog file path")
	cmd.Flags().StringVarP(&outPath, "savepath", "o", "", "output excel path")
	cmd.Flags().Int64Var(&startTime, "starttime", 0, "start unix timestamp")
	cmd.Flags().Int64Var(&endTime, "endtime", 0, "end unix timestamp")
	cmd.Flags().BoolVarP(&vbo, "verbose", "v", false, "show the detail verbose")
	cmd.Flags().BoolVarP(&skipquery, "skip-query", "s", true, "skip query write to xlsx like BEGIN COMMIT")
	cmd.Flags().IntVar(&bigThan, "big", 0, "show tx big than x bytes")
	cmd.Flags().IntVar(&smallThan, "small", 0, "show tx small than x bytes")
	cmd.Flags().StringVar(&prefix, "prefix", "mysql-bin", "mysql binlog prefix")
}

var cmd = &cobra.Command{
	Use:   "",
	Short: "binlog parser",
	Long:  "binlog parser",
	Run: func(cmd *cobra.Command, args []string) {
		if len(filePath) == 0 {
			starlog.Warningln("Please enter a binlog path or folder")
			return
		}
		ParseBinlog()
	},
}

func main() {
	cmd.Execute()
}

func ParseBinlog() {
	var err error
	foundCount := 0
	var totalGtid *gtid.Gtid
	var owrt *xlsx.File
	var res *xlsx.Sheet
	if outPath != "" {
		owrt = xlsx.NewFile()
		res, err = owrt.AddSheet("结果")
		if err != nil {
			starlog.Errorln(err)
			return
		}
		title := res.AddRow()
		title.AddCell().SetValue("序号")
		title.AddCell().SetValue("GTID")
		title.AddCell().SetValue("时间")
		title.AddCell().SetValue("时间戳")
		title.AddCell().SetValue("StartPos")
		title.AddCell().SetValue("EndPos")
		title.AddCell().SetValue("事务大小")
		title.AddCell().SetValue("影响行数")
		title.AddCell().SetValue("单语句StartPos")
		title.AddCell().SetValue("单语句EndPos")
		title.AddCell().SetValue("单语句时间")
		title.AddCell().SetValue("单语句影响行数")
		title.AddCell().SetValue("单语句影响库")
		title.AddCell().SetValue("单语句影响表")
		title.AddCell().SetValue("SQL类型")
		title.AddCell().SetValue("具体SQL")
		title.AddCell().SetValue("变更内容")
		res.SetColWidth(0, 0, 5)
		res.SetColWidth(1, 1, 40)
		res.SetColWidth(3, 6, 6)
		res.SetColWidth(7, 7, 5)
		res.SetColWidth(15, 15, 40)
		owrt.Save(outPath)
	}
	getParser := func(fpath string) string {
		var sTime, eTime time.Time
		if startTime != 0 {
			sTime = time.Unix(startTime, 0)
		}
		if endTime != 0 {
			eTime = time.Unix(endTime, 0)
		}
		var filter = binlog.BinlogFilter{
			IncludeGtid: includeGtid,
			ExcludeGtid: excludeGtid,
			StartPos:    startPos,
			EndPos:      endPos,
			StartDate:   sTime,
			EndDate:     eTime,
			BigThan:     bigThan,
			SmallThan:   smallThan,
		}
		var cGtid *gtid.Gtid
		err = binlog.ParseBinlogWithFilter(fpath, pos, filter, func(tx binlog.Transaction) {
			foundCount++
			if cGtid == nil {
				cGtid, _ = gtid.Parse(tx.GTID)
			} else {
				cGtid.Add(tx.GTID)
			}
			if totalGtid == nil {
				totalGtid, _ = gtid.Parse(tx.GTID)
			} else {
				totalGtid.Add(tx.GTID)
			}
			if !vbo {
				fmt.Printf("已找到%d个合规GTID\r", foundCount)
			} else {
				fmt.Printf("GTID:%s Time:%s StartPos:%v EndPos:%v RowsCount:%v Size:%v Detail:%+v\n",
					tx.GTID, tx.Time, tx.StartPos, tx.EndPos, tx.RowsCount, tx.Size, tx.Txs)
			}
			if outPath != "" {
				for _, t := range tx.Txs {
					if skipquery && (strings.ToLower(t.Sql) == "begin" || strings.ToLower(t.Sql) == "commit") {
						continue
					}
					r := res.AddRow()
					r.AddCell().SetValue(foundCount)
					r.AddCell().SetValue(tx.GTID)
					r.AddCell().SetValue(tx.Time.String())
					r.AddCell().SetValue(tx.Timestamp)
					r.AddCell().SetValue(tx.StartPos)
					r.AddCell().SetValue(tx.EndPos)
					r.AddCell().SetValue(tx.Size)
					r.AddCell().SetValue(tx.RowsCount)
					r.AddCell().SetValue(t.StartPos)
					r.AddCell().SetValue(t.EndPos)
					r.AddCell().SetValue(t.Time.String())
					r.AddCell().SetValue(t.RowCount)
					r.AddCell().SetValue(t.Db)
					r.AddCell().SetValue(t.Table)
					r.AddCell().SetValue(t.SqlType)
					r.AddCell().SetValue(t.Sql)
					r.AddCell().SetValue(t.Rows)
				}
			}
		})
		var cGtidStr string
		if cGtid != nil {
			cGtidStr = cGtid.String()
		}
		if outPath != "" {
			err = owrt.Save(outPath)
			if err != nil {
				starlog.Errorln(err)
				return cGtidStr
			}
		}
		if err != nil {
			starlog.Errorln(err)
			return cGtidStr
		}
		return cGtidStr
	}
	var gtidRes [][]string
	for _, fp := range filePath {
		if staros.IsFolder(fp) {
			files, err := os.ReadDir(fp)
			if err != nil {
				starlog.Errorln("read folder failed:", err)
				return
			}
			rgp := regexp.MustCompile(`^` + prefix + `\.\d+$`)
			for _, v := range files {
				if !v.IsDir() && rgp.MatchString(v.Name()) {
					gtidRes = append(gtidRes, []string{v.Name(), getParser(filepath.Join(fp, v.Name()))})
				}
			}
		} else {
			getParser(fp)
		}
	}
	if outPath != "" {
		owrt.Save(outPath)
	}
	fmt.Println("")
	if len(gtidRes) != 0 {
		for _, v := range gtidRes {
			fmt.Printf("%s:%s\n", v[0], v[1])
		}
	}
	fmt.Println("")
	allGtid := ""
	if totalGtid != nil {
		allGtid = totalGtid.String()
	}
	fmt.Printf("Total Gtid:%v\nTotal Binlog Number:%v\n", allGtid, foundCount)
}