status info

go.mod

@@ -3,11 +3,11 @@ module git.wecise.com/wecise/cgimport
 go 1.22.0
 
 require (
-	git.wecise.com/wecise/odb-go v0.0.0-20250220112239-ec74e16da8a4
+	git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac
 	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da
 	github.com/scylladb/go-set v1.0.2
 	github.com/spf13/cast v1.7.1
-	github.com/wecisecode/util v0.0.9
+	github.com/wecisecode/util v0.1.1
 )
 
 require (

go.sum

@@ -1,5 +1,5 @@
-git.wecise.com/wecise/odb-go v0.0.0-20250220112239-ec74e16da8a4 h1:X6yFjxoLFQL1uP1ENNvuvYff9uvL+KbCVxJkmzeV0UQ=
-git.wecise.com/wecise/odb-go v0.0.0-20250220112239-ec74e16da8a4/go.mod h1:Nt8CbHMuLK+eTEn6XCCEyaG5OT0HNjbfzIMsnTaQIPk=
+git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac h1:12h8OLjgLCvW5733EkAfgqtGF3raR2WlxcQOdvO2Xqg=
+git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac/go.mod h1:kb5NJtgDjnsd67N78NMB5Nyoi5fAirQ2dvxeJ2gMbCo=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
 github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
@@ -73,8 +73,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/wecisecode/util v0.0.9 h1:YBAk0KCEnUWLpWycSP1DarNdq2qpEyNVqUmAwfFHhNo=
-github.com/wecisecode/util v0.0.9/go.mod h1:+WVChwgmcZMABsWtjhlyjmezG4U6gGRGOy2Es3zB6Qw=
+github.com/wecisecode/util v0.1.1 h1:27Nv2apMYNdOExS0HkN6r9G5S0XZjUovlSZllxLQdJ8=
+github.com/wecisecode/util v0.1.1/go.mod h1:fASrRbyMPjssR1owWRMZIBCUSRYo5YkcxOk+kG59byY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs=

importer/importer.go

@@ -135,7 +135,7 @@ const (
 func (importer *Importer) ImportFiles(fwop func(basedir string, fpath string) FWOP, logstatus bool) (filescount, linescount, recordscount, retrycount int64, usetime time.Duration, err error) {
 	// 遍历文件目录
 	var wg sync.WaitGroup
-	fw, e := filewalker.NewFileWalker([]string{importer.datapath}, ".*")
+	fw, e := filewalker.NewFileWalker([]string{importer.datapath}, `^[^\.].*`)
 	if e != nil {
 		err = e
 		return
@@ -265,6 +265,8 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 	skiplines := int(linefrom)
 	blockcount = blockfrom
 	doinglines := []int64{}
+	donelines := linefrom
+	doneblocks := blockfrom
 	retrycount = totalretrycount
 	// maxresponsetime := time.Duration(0)
 	var wg sync.WaitGroup
@@ -313,7 +315,9 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 				err = merrs.NewError(e, merrs.SSMaps{{"filename": filename}, {"linecount": fmt.Sprint(doingline)}, {"line": line}})
 				return
 			}
+			atomic.AddInt64(&doneblocks, 1)
 			if logstatus {
+				readinglines := doinglines[len(doinglines)-1]
 				if doingline == doinglines[0] {
 					importer.importstatus.mutex.Lock()
 					importer.importstatus.ImportStatus[filename] = &ImportStatus{
@@ -321,14 +325,15 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 						RecordsCount: doingblock,
 						RetryCount:   retrycount,
 					}
+					donelines = doingline
 					importer.importstatus.TotalUseTime = time.Since(importer.starttime)
 					importer.importstatus.Save()
 					doinglines = doinglines[1:]
 					if time.Since(lastlogtime) > 5*time.Second {
 						if retrycount > 0 {
-							logger.Info("file", filename, "imported", doingline, "lines", doingblock, "records", retrycount, "retry times")
+							logger.Info("file", filename, "read", readinglines, "lines,", "importing", len(doinglines), "chunks,", "imported", donelines, "lines", doneblocks, "records", retrycount, "retry times")
 						} else {
-							logger.Info("file", filename, "imported", doingline, "lines", doingblock, "records")
+							logger.Info("file", filename, "read", readinglines, "lines,", "importing", len(doinglines), "chunks,", "imported", donelines, "lines", doneblocks, "records")
 						}
 						lastlogtime = time.Now()
 					}
@@ -337,8 +342,17 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 					for i, l := range doinglines {
 						if l == doingline {
 							doinglines = append(doinglines[:i], doinglines[i+1:]...)
+							break
 						}
 					}
+					if time.Since(lastlogtime) > 5*time.Second {
+						if retrycount > 0 {
+							logger.Info("file", filename, "read", readinglines, "lines,", "importing", len(doinglines), "chunks,", "imported", donelines, "lines", doneblocks, "records", retrycount, "retry times")
+						} else {
+							logger.Info("file", filename, "read", readinglines, "lines,", "importing", len(doinglines), "chunks,", "imported", donelines, "lines", doneblocks, "records")
+						}
+						lastlogtime = time.Now()
+					}
 				}
 			}
 		})

importer/odbcimporter.go

@@ -295,7 +295,10 @@ var odbcretryconfig []*ODBCRetryConfig
 
 func init() {
 	mcfg.OnChange(func() {
-		_odbcretry := mcfg.GetStrings("odbc.retry", "-1, 1s, timed out", "-1, 1s, proc timeout")
+		_odbcretry := mcfg.GetStrings("odbc.retry",
+			"-1, 1s, timed out",
+			"-1, 1s, proc timeout",
+			"-1, 1m, no response received from cassandra")
 		if strings.Join(_odbcretry, "|") != odbcretry {
 			odbcretryconfig = RetryConfig(_odbcretry...)
 			odbcretry = strings.Join(_odbcretry, "|")

odbc/cfg.go

@@ -21,7 +21,7 @@ file=` + filepath.Join(DefaultAppName, "log.log") + `
 console=true
 color=true
 consolelevel=info
-format=yyyy-MM-dd HH:mm:ss.SSSSSS [pid] [level] module/file:line msg
+format=yyyy-MM-dd HH:mm:ss.SSSSSS [level] msg
 eol=\n
 size=10MB
 count=100
@@ -36,10 +36,6 @@ var Logger = ulog.New().WithConfig(Config, "log")
 var LogFile = Logger.FileOutPath()
 var LogDebug = Logger.FileOutLevel() <= ulog.DEBUG
 
-func init() {
-	Logger.SetFormat("yyyy-MM-dd HH:mm:ss.SSSSSS [pid] [level] msg", "\n")
-}
-
 const (
 	DP_READFILE = 1 << iota
 	DP_PROCESSCONTINUE