config logger

cfg/cfg.go

@@ -0,0 +1,35 @@
+package cfg
+
+import (
+	"fmt"
+	"path/filepath"
+
+	ucfg "git.wecise.com/wecise/util/cfg"
+	ulog "git.wecise.com/wecise/util/logger"
+)
+
+var Config = ucfg.MConfig(&ucfg.CfgOption{
+	Name: "m:default",
+	Type: ucfg.INI_TEXT,
+	Values: []string{`
+[log]
+level=debug
+dir=` + filepath.Join("/", "opt", "matrix", "var", "logs") + `
+file=` + "cgimport.log" + `
+console=true
+color=true
+consolelevel=info
+format=yyyy-MM-dd HH:mm:ss.SSSSSS [pid] [level] module/file:line msg
+eol=\n
+size=10MB
+count=100
+scroll=1d
+expire=14d
+`}},
+	ucfg.GetIniFileCfgOption(filepath.Join(fmt.Sprint("cgimport", ".conf"))),
+	ucfg.GetIniFileCfgOption(filepath.Join(fmt.Sprint(ucfg.DefaultAppName, ".conf"))),
+	ucfg.CFGOPTION_ENVS,
+	ucfg.CFGOPTION_ARGS)
+
+var Logger = ulog.New().WithConfig(Config, "log")
+var LogFile = Logger.FileOutPath()

cgf/cgf.go

@@ -9,12 +9,16 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"git.wecise.com/wecise/cgimport/cfg"
 	"git.wecise.com/wecise/cgimport/cgf/reader"
 	"git.wecise.com/wecise/util/filewalker"
 	"git.wecise.com/wecise/util/merrs"
 	"git.wecise.com/wecise/util/rc"
 )
 
+var mcfg = cfg.Config
+var logger = cfg.Logger
+
 func ImportDir(datapath string, parallel int) (filescount, recordscount int64, err error) {
 	// 遍历文件目录
 	var cgirc = rc.NewRoutinesController("", parallel)
@@ -61,12 +65,19 @@ func ImportFile(filepath string) (blockcount int, err error) {
 	return importReader(filepath, f)
 }
 
+var parserc = rc.NewRoutinesController("", 1000)
+
 func importReader(filename string, buf io.Reader) (blockcount int, err error) {
 	br, e := reader.NewBlockReader(filename, buf)
 	if e != nil {
 		return blockcount, merrs.NewError(e, merrs.SSMaps{{"filename": filename}})
 	}
+	var wg sync.WaitGroup
+	defer wg.Wait()
 	for {
+		if err != nil {
+			break
+		}
 		block, linecount, e := br.ReadBlock()
 		if e != nil {
 			return blockcount, merrs.NewError(e, merrs.SSMaps{{"filename": filename}, {"line": fmt.Sprint(linecount)}})
@@ -74,12 +85,21 @@ func importReader(filename string, buf io.Reader) (blockcount int, err error) {
 		if block == nil {
 			return
 		}
-		e = importBlock(block, filename, linecount)
+		wg.Add(1)
+		e = parserc.ConcurCall(1, func() {
+			defer wg.Done()
+			e = importBlock(block, filename, linecount)
+			if e != nil {
+				err = merrs.NewError(e, merrs.SSMaps{{"filename": filename}, {"line": fmt.Sprint(linecount)}})
+				return
+			}
+			blockcount++
+		})
 		if e != nil {
 			return blockcount, merrs.NewError(e, merrs.SSMaps{{"filename": filename}, {"line": fmt.Sprint(linecount)}})
 		}
-		blockcount++
 	}
+	return
 }
 
 func importBlock(block map[string]any, filename string, linecount int) (err error) {
@@ -87,6 +107,6 @@ func importBlock(block map[string]any, filename string, linecount int) (err erro
 	if e != nil {
 		return e
 	}
-	fmt.Println(fmt.Sprint("import ", filename, "[", linecount, "]:", string(bs)))
+	logger.Debug(fmt.Sprint("import ", filename, "[", linecount, "]:", string(bs)))
 	return
 }

go.mod

@@ -2,7 +2,10 @@ module git.wecise.com/wecise/cgimport
 
 go 1.20
 
-require git.wecise.com/wecise/util v0.0.0-20250123073753-71dedf03772d
+require (
+	git.wecise.com/wecise/util v0.0.0-20250204084114-10f3a56ec32b
+	github.com/spf13/cast v1.7.0
+)
 
 require (
 	github.com/coreos/go-semver v0.3.0 // indirect
@@ -13,7 +16,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/spacemonkeygo/errors v0.0.0-20201030155909-2f5f890dbc62 // indirect
-	github.com/spf13/cast v1.7.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.8 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.8 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.8 // indirect

go.sum

@@ -1,5 +1,5 @@
-git.wecise.com/wecise/util v0.0.0-20250123073753-71dedf03772d h1:A+ez5eh0v8Sz5BDbi0nysRF0dROFkPlM+jnZEk81b5Y=
-git.wecise.com/wecise/util v0.0.0-20250123073753-71dedf03772d/go.mod h1:2YXWE9m5mNgAu40zpYrL3woGz6S8CoHAW/CJeWXaIko=
+git.wecise.com/wecise/util v0.0.0-20250204084114-10f3a56ec32b h1:GQ5IfKNb+XBziRCd0PmkBlteXf1upyp4sNhpAZS3UZE=
+git.wecise.com/wecise/util v0.0.0-20250204084114-10f3a56ec32b/go.mod h1:2YXWE9m5mNgAu40zpYrL3woGz6S8CoHAW/CJeWXaIko=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=

main.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	"git.wecise.com/wecise/cgimport/cfg"
 	"git.wecise.com/wecise/cgimport/cgf"
-	"git.wecise.com/wecise/util/cfg"
 )
 
 // 获取配置信息
@@ -14,7 +14,8 @@ import (
 //		当前工作目录下 与应用同名的 .conf 文件
 //		环境变量
 //		命令行参数
-var mcfg = cfg.MConfig()
+var mcfg = cfg.Config
+var logger = cfg.Logger
 
 func main() {
 	// 开始计时
@@ -35,4 +36,5 @@ func main() {
 	}
 	// 输出统计信息
 	fmt.Println("total import", filescount, "files", recordscount, "records", "in", time.Since(st))
+	fmt.Println("access", cfg.LogFile, "for detail information")
 }