package main import ( "b612.me/mysql/binlog" "b612.me/mysql/gtid" "b612.me/starlog" "b612.me/staros" "fmt" "github.com/spf13/cobra" "github.com/tealeg/xlsx" "os" "path/filepath" "regexp" "strings" "time" ) var ( bigThan int smallThan int startPos int endPos int startTime int64 endTime int64 includeGtid string excludeGtid string filePath []string outPath string vbo bool skipquery bool pos int64 prefix string ) func init() { cmd.Flags().IntVarP(&endPos, "pos", "P", 0, "skipPos of binlog") cmd.Flags().IntVarP(&startPos, "start-pos", "S", 0, "startPos of binlog") cmd.Flags().IntVarP(&endPos, "end-pos", "E", 0, "endPos of binlog") cmd.Flags().StringVarP(&includeGtid, "include-gtid", "i", "", "include gtid") cmd.Flags().StringVarP(&excludeGtid, "exclude-gtid", "e", "", "exclude gtid") cmd.Flags().StringSliceVarP(&filePath, "path", "p", []string{}, "binlog file path") cmd.Flags().StringVarP(&outPath, "savepath", "o", "", "output excel path") cmd.Flags().Int64Var(&startTime, "starttime", 0, "start unix timestamp") cmd.Flags().Int64Var(&endTime, "endtime", 0, "end unix timestamp") cmd.Flags().BoolVarP(&vbo, "verbose", "v", false, "show the detail verbose") cmd.Flags().BoolVarP(&skipquery, "skip-query", "s", true, "skip query write to xlsx like BEGIN COMMIT") cmd.Flags().IntVar(&bigThan, "big", 0, "show tx big than x bytes") cmd.Flags().IntVar(&smallThan, "small", 0, "show tx small than x bytes") cmd.Flags().StringVar(&prefix, "prefix", "mysql-bin", "mysql binlog prefix") } var cmd = &cobra.Command{ Use: "", Short: "binlog parser", Long: "binlog parser", Run: func(cmd *cobra.Command, args []string) { if len(filePath) == 0 { starlog.Warningln("Please enter a binlog path or folder") return } now := time.Now() ParseBinlog() cost := time.Now().Sub(now).Seconds() fmt.Println("") fmt.Printf("Time Cost:%.2fs", cost) }, } func main() { cmd.Execute() } func ParseBinlog() { var err error foundCount := 0 var totalGtid *gtid.Gtid var owrt *xlsx.File var res *xlsx.Sheet if outPath != "" { owrt = xlsx.NewFile() res, err = owrt.AddSheet("结果") if err != nil { starlog.Errorln(err) return } title := res.AddRow() title.AddCell().SetValue("序号") title.AddCell().SetValue("GTID") title.AddCell().SetValue("时间") title.AddCell().SetValue("时间戳") title.AddCell().SetValue("StartPos") title.AddCell().SetValue("EndPos") title.AddCell().SetValue("事务大小") title.AddCell().SetValue("影响行数") title.AddCell().SetValue("压缩类型") title.AddCell().SetValue("单语句StartPos") title.AddCell().SetValue("单语句EndPos") title.AddCell().SetValue("单语句时间") title.AddCell().SetValue("单语句影响行数") title.AddCell().SetValue("单语句影响库") title.AddCell().SetValue("单语句影响表") title.AddCell().SetValue("SQL类型") title.AddCell().SetValue("具体SQL") title.AddCell().SetValue("变更内容") res.SetColWidth(0, 0, 5) res.SetColWidth(1, 1, 40) res.SetColWidth(3, 6, 6) res.SetColWidth(7, 7, 5) res.SetColWidth(16, 16, 40) owrt.Save(outPath) } getParser := func(fpath string) string { var sTime, eTime time.Time if startTime != 0 { sTime = time.Unix(startTime, 0) } if endTime != 0 { eTime = time.Unix(endTime, 0) } onlyGtid := false if !vbo && outPath == "" { onlyGtid = true } var filter = binlog.BinlogFilter{ IncludeGtid: includeGtid, ExcludeGtid: excludeGtid, StartPos: startPos, EndPos: endPos, StartDate: sTime, EndDate: eTime, BigThan: bigThan, SmallThan: smallThan, OnlyShowGtid: onlyGtid, } var cGtid *gtid.Gtid proc := make(chan string, 1000) if !vbo { go func() { var latest string var count = uint16(0) for { select { case tmp := <-proc: if tmp == "end" { fmt.Println(latest) return } latest = tmp if count%10 == 0 { fmt.Print(latest) } count++ case <-time.After(time.Second * 2): fmt.Print(latest) } } }() } err = binlog.ParseBinlogWithFilter(fpath, pos, filter, func(tx binlog.Transaction) { foundCount++ if cGtid == nil { cGtid, _ = gtid.Parse(tx.GTID) } else { cGtid.Add(tx.GTID) } if totalGtid == nil { totalGtid, _ = gtid.Parse(tx.GTID) } else { totalGtid.Add(tx.GTID) } if !vbo { proc <- fmt.Sprintf("已找到%d个合法GTID\r", foundCount) } else { fmt.Printf("GTID:%s Time:%s StartPos:%v EndPos:%v RowsCount:%v Size:%v Detail:%+v\n", tx.GTID, tx.Time, tx.StartPos, tx.EndPos, tx.RowsCount, tx.Size, tx.Txs) } if outPath != "" { for _, t := range tx.Txs { if skipquery && (strings.ToLower(t.Sql) == "begin" || strings.ToLower(t.Sql) == "commit") { continue } r := res.AddRow() r.AddCell().SetValue(foundCount) r.AddCell().SetValue(tx.GTID) r.AddCell().SetValue(tx.Time.String()) r.AddCell().SetValue(tx.Timestamp) r.AddCell().SetValue(tx.StartPos) r.AddCell().SetValue(tx.EndPos) r.AddCell().SetValue(tx.Size) r.AddCell().SetValue(tx.RowsCount) if t.CompressionType == "" { r.AddCell().SetValue("NONE") } else { r.AddCell().SetValue(t.CompressionType) } r.AddCell().SetValue(t.StartPos) r.AddCell().SetValue(t.EndPos) r.AddCell().SetValue(t.Time.String()) r.AddCell().SetValue(t.RowCount) r.AddCell().SetValue(t.Db) r.AddCell().SetValue(t.Table) r.AddCell().SetValue(t.SqlType) r.AddCell().SetValue(t.Sql) r.AddCell().SetValue(t.Rows) } } }) if !vbo { time.Sleep(time.Millisecond * 500) } var cGtidStr string if cGtid != nil { cGtidStr = cGtid.String() } if outPath != "" { err = owrt.Save(outPath) if err != nil { starlog.Errorln(err) return cGtidStr } } if err != nil { starlog.Errorln(err) return cGtidStr } return cGtidStr } var gtidRes [][]string for _, fp := range filePath { if staros.IsFolder(fp) { files, err := os.ReadDir(fp) if err != nil { starlog.Errorln("read folder failed:", err) return } rgp := regexp.MustCompile(`^` + prefix + `\.\d+$`) for _, v := range files { if !v.IsDir() && rgp.MatchString(v.Name()) { gtidRes = append(gtidRes, []string{v.Name(), getParser(filepath.Join(fp, v.Name()))}) } } } else { getParser(fp) } } if outPath != "" { owrt.Save(outPath) } fmt.Println("") if len(gtidRes) != 0 { for _, v := range gtidRes { fmt.Printf("%s:%s\n", v[0], v[1]) } } fmt.Println("") allGtid := "" if totalGtid != nil { allGtid = totalGtid.String() } fmt.Printf("Total Gtid:%v\nTotal Binlog Number:%v\n", allGtid, foundCount) }