format code

cgimport.conf

@@ -1,11 +1,43 @@
 
 [log]
+; 日志相关配置参数
+; level=trace        ; 日志级别 trace，debug，info，warn，error，fatal，默认 trace
+; console=true       ; 是否控制台输出，默认 true
+; color=true         ; 控制台输出是否根据级别区分颜色，默认 true，需要控制台客户端支持
+; consolelevel=info  ; 控制台显示级别，-1 跟随主级别定义，默认 info
+; format=            ; 默认 yyyy-MM-dd HH:mm:ss.SSSSSS [level] msg
+; eol=\r\n           ; 默认 \n
+; dir=               ; 默认 /opt/matrix/var/logs
+; file=              ; 默认 cgimport/log.log
+; size=5m            ; 尺寸滚动，默认 10MB，支持存储单位 byte kb mb gb tb pb eb， 默认为 byte
+; count=20           ; 保留数量，默认 100
+; scroll=1天         ; 时间滚动 scroll，覆盖 dialy 设置，默认 1 天，支持时间单位 d 天 h 小时 m 分钟 s 秒 ms 毫秒 us 微秒 ns 纳秒，默认毫秒
+; expire=14d         ; 保留时间，默认 14 天
 level=debug
 
 [cgi]
-datapath=
-parallel=20
+# 文件解析并发数
+#parallel=20
+# 指定数据文件路径，默认 data
+#datapath=
+
+# 类名与文件名对应关系，文件名支持通配符，以下为默认配置
+mapping.class.master=*MASTER*.csv
+mapping.class.level1=*_L1_*
+mapping.class.level2=*_L2_*
+mapping.class.level3=*_L3_*
+mapping.class.level4=*_L4_*
+mapping.class.level5=*_L5_*
+mapping.class.level6=*_L6_*
+mapping.class.level7=*_L7_*
+mapping.class.level8=*_L8_*
+# EDGE 与普通类不同，只用于 edge 数据
+mapping.class.EDGE=*EDGE*.csv
 
 [odbc]
+;指定odbserver路径
+odbpath=127.0.0.1:11001
+;指定keyspace
+keyspace=matrix
+# 数据插入请求并发数限制
 concurrent.limit=20
-

go.mod

@@ -3,11 +3,11 @@ module git.wecise.com/wecise/cgimport
 go 1.22.0
 
 require (
-	git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac
+	git.wecise.com/wecise/odb-go v0.0.0-20250224095700-05adc4d30431
 	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da
 	github.com/scylladb/go-set v1.0.2
 	github.com/spf13/cast v1.7.1
-	github.com/wecisecode/util v0.1.1
+	github.com/wecisecode/util v0.1.2
 )
 
 require (

go.sum

@@ -1,5 +1,5 @@
-git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac h1:12h8OLjgLCvW5733EkAfgqtGF3raR2WlxcQOdvO2Xqg=
-git.wecise.com/wecise/odb-go v0.0.0-20250224012237-387f83aad2ac/go.mod h1:kb5NJtgDjnsd67N78NMB5Nyoi5fAirQ2dvxeJ2gMbCo=
+git.wecise.com/wecise/odb-go v0.0.0-20250224095700-05adc4d30431 h1:VTCFyFviYSMmDLlYNDvLdxAQfpupVzgz6rjO1cHHoto=
+git.wecise.com/wecise/odb-go v0.0.0-20250224095700-05adc4d30431/go.mod h1:xIY+Dmw2LK9ephqNs8r8clD6zQ5+BuaTgk+6TxEe15M=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
 github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
@@ -73,8 +73,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/wecisecode/util v0.1.1 h1:27Nv2apMYNdOExS0HkN6r9G5S0XZjUovlSZllxLQdJ8=
-github.com/wecisecode/util v0.1.1/go.mod h1:fASrRbyMPjssR1owWRMZIBCUSRYo5YkcxOk+kG59byY=
+github.com/wecisecode/util v0.1.2 h1:ELBM95ZptoY/suWdWWn07rXXl56v5KLC4TvqQFibGag=
+github.com/wecisecode/util v0.1.2/go.mod h1:fASrRbyMPjssR1owWRMZIBCUSRYo5YkcxOk+kG59byY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs=

importer/importer.go

@@ -17,8 +17,10 @@ import (
 	"git.wecise.com/wecise/cgimport/odbc"
 	"git.wecise.com/wecise/cgimport/reader"
 	"git.wecise.com/wecise/cgimport/schema"
+	"github.com/wecisecode/util/cmap"
 	"github.com/wecisecode/util/filewalker"
 	"github.com/wecisecode/util/merrs"
+	"github.com/wecisecode/util/pattern"
 	"github.com/wecisecode/util/rc"
 )
 
@@ -230,34 +232,46 @@ func (importer *Importer) ImportFile(filepath string, linefrom, blockfrom, total
 	return importer.importReader(filepath, f, linefrom, blockfrom, totalretrycount, logstatus)
 }
 
+var fileclassmapping = cmap.NewSingle[string, *regexp.Regexp]()
+
+func init() {
+	fileclassmapping.Set("EDGE", regexp.MustCompile(pattern.Contain2RegexpString(`EDGE`)))
+	fileclassmapping.Set("master", regexp.MustCompile(pattern.Contain2RegexpString(`MASTER`)))
+	fileclassmapping.Set("level1", regexp.MustCompile(pattern.Contain2RegexpString(`_L1_`)))
+	fileclassmapping.Set("level2", regexp.MustCompile(pattern.Contain2RegexpString(`_L2_`)))
+	fileclassmapping.Set("level3", regexp.MustCompile(pattern.Contain2RegexpString(`_L3_`)))
+	fileclassmapping.Set("level4", regexp.MustCompile(pattern.Contain2RegexpString(`_L4_`)))
+	fileclassmapping.Set("level5", regexp.MustCompile(pattern.Contain2RegexpString(`_L5_`)))
+	fileclassmapping.Set("level6", regexp.MustCompile(pattern.Contain2RegexpString(`_L6_`)))
+	fileclassmapping.Set("level7", regexp.MustCompile(pattern.Contain2RegexpString(`_L7_`)))
+	fileclassmapping.Set("level8", regexp.MustCompile(pattern.Contain2RegexpString(`_L8_`)))
+	keys := mcfg.Keys()
+	for _, key := range keys {
+		if strings.HasPrefix(key, "mapping.class.") {
+			classname := key[len("mapping.class."):]
+			filepatterns := mcfg.GetStrings(key)
+			for _, fp := range filepatterns {
+				fp = pattern.Wildcard2RegexpString(fp)
+				fileclassmapping.Set(classname, regexp.MustCompile(fp))
+			}
+		}
+	}
+}
+
 func (importer *Importer) importReader(filename string, buf io.Reader, linefrom, blockfrom, totalretrycount int64, logstatus bool) (linecount, blockcount, retrycount int64, err error) {
-	var filetype schema.FileType
-	switch {
-	case strings.Contains(filename, "_L1_"):
-		filetype = schema.FT_LEVEL1
-	case strings.Contains(filename, "_L2_"):
-		filetype = schema.FT_LEVEL2
-	case strings.Contains(filename, "_L3_"):
-		filetype = schema.FT_LEVEL3
-	case strings.Contains(filename, "_L4_"):
-		filetype = schema.FT_LEVEL4
-	case strings.Contains(filename, "_L5_"):
-		filetype = schema.FT_LEVEL5
-	case strings.Contains(filename, "_L6_"):
-		filetype = schema.FT_LEVEL6
-	case strings.Contains(filename, "_L7_"):
-		filetype = schema.FT_LEVEL7
-	case strings.Contains(filename, "_L8_"):
-		filetype = schema.FT_LEVEL8
-	case strings.Contains(filename, "MASTER"):
-		filetype = schema.FT_MASTER
-	case strings.Contains(filename, "EDGE"):
-		filetype = schema.FT_EDGE
-	default:
-		err = merrs.NewError("filename does not conform to the agreed format " + filename)
+	classname := ""
+	fileclassmapping.Fetch(func(key string, v *regexp.Regexp) bool {
+		if v.MatchString(filename) {
+			classname = key
+			return false
+		}
+		return true
+	})
+	if classname == "" {
+		err = merrs.NewError("cannot mapping to class, filename:" + filename)
 		return
 	}
-	br, e := reader.NewBlockReader(filename, filetype, buf)
+	br, e := reader.NewBlockReader(filename, classname, buf)
 	if e != nil {
 		return linecount, blockcount, retrycount, merrs.NewError(e, merrs.SSMaps{{"filename": filename}})
 	}
@@ -309,7 +323,7 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 			// 	"RQ:", importer.fileimportrc.QueueCount(),
 			// 	"WQ:", importer.odbcqueryrc.QueueCount(),
 			// 	"maxresponsetime:", maxresponsetime)
-			rc, e := importer.importRecord(block, line, filename, filetype, int(doingline))
+			rc, e := importer.importRecord(block, line, filename, classname, int(doingline))
 			atomic.AddInt64(&retrycount, int64(rc))
 			if e != nil {
 				err = merrs.NewError(e, merrs.SSMaps{{"filename": filename}, {"linecount": fmt.Sprint(doingline)}, {"line": line}})
@@ -362,7 +376,7 @@ func (importer *Importer) importReader(filename string, buf io.Reader, linefrom,
 	}
 }
 
-func (importer *Importer) importRecord(record map[string]any, line string, filename string, filetype schema.FileType, linecount int) (retrycount int, err error) {
+func (importer *Importer) importRecord(record map[string]any, line string, filename string, classaliasname string, linecount int) (retrycount int, err error) {
 	if odbc.LogDebug {
 		bs, e := json.MarshalIndent(record, "", "  ")
 		if e != nil {
@@ -370,12 +384,10 @@ func (importer *Importer) importRecord(record map[string]any, line string, filen
 		}
 		logger.Trace(fmt.Sprint("import ", filename, "[", linecount, "]:", string(bs)))
 	}
-	var classaliasname string
-	switch filetype {
-	case schema.FT_EDGE:
+	switch classaliasname {
+	case schema.EDGE:
 		graph.CacheEdgeInfo(record)
 	default:
-		classaliasname = string(filetype)
 		retrycount, err = importer.odbcimporter.InsertData(classaliasname, record)
 		if err != nil {
 			err = merrs.NewError(err, merrs.SSMaps{{"filename": filename}, {"linecount": fmt.Sprint(linecount)}, {"line": line}})

odbc/odbclient.go

@@ -21,9 +21,9 @@ var default_odbpaths = `127.0.0.1:11001`
 
 var keyspace string
 var default_config = &odb.Config{
-	Hosts:    []string{"127.0.0.1:11001"}, // 47.92.151.165:11001
-	Keyspace: "oktest",
-	User:     fmt.Sprint("测试客户端"),
+	Hosts:    []string{default_odbpaths},
+	Keyspace: default_keyspace,
+	User:     DefaultAppName,
 	Pass:     "********",
 	PoolSize: 20,
 	Debug:    false,

reader/blockreader.go

@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 
 	"git.wecise.com/wecise/cgimport/odbc"
-	"git.wecise.com/wecise/cgimport/schema"
 	"github.com/wecisecode/util/merrs"
 )
 
@@ -16,13 +15,13 @@ type BlockReader interface {
 	ReadBlock(skiplines int) (block map[string]any, line string, linecount int, err error)
 }
 
-func NewBlockReader(filename string, filetype schema.FileType, reader io.Reader) (BlockReader, error) {
+func NewBlockReader(filename string, classname string, reader io.Reader) (BlockReader, error) {
 	fileext := filepath.Ext(filename)
 	switch fileext {
 	case ".csv":
-		return NewCSVBlockReader(filename, filetype, reader), nil
+		return NewCSVBlockReader(filename, classname, reader), nil
 	case ".txt":
-		return NewTXTBlockReader(filename, filetype, reader), nil
+		return NewTXTBlockReader(filename, classname, reader), nil
 	}
-	return nil, merrs.UnsupportedError.NewError("unsupported data format " + filetype)
+	return nil, merrs.UnsupportedError.NewError("unsupported data format " + fileext)
 }

reader/csvreader.go

@@ -13,19 +13,19 @@ import (
 
 type CSVBlockReader struct {
 	*LineReader
-	filetype schema.FileType
-	csvkeys  []string
+	classname string
+	csvkeys   []string
 }
 
-func NewCSVBlockReader(filename string, filetype schema.FileType, reader io.Reader) *CSVBlockReader {
+func NewCSVBlockReader(filename string, classname string, reader io.Reader) *CSVBlockReader {
 	return &CSVBlockReader{
 		LineReader: NewLineReader(filename, reader),
-		filetype:   filetype,
+		classname:  classname,
 	}
 }
 
 func (br *CSVBlockReader) ReadBlock(skiplines int) (block map[string]any, line string, linecount int, err error) {
-	classname := string(br.filetype)
+	classname := br.classname
 	ci := schema.ClassInfos.GetIFPresent(classname)
 	eof := false
 	for {

reader/txtreader.go

@@ -6,21 +6,19 @@ import (
 	"io"
 	"regexp"
 	"strings"
-
-	"git.wecise.com/wecise/cgimport/schema"
 )
 
 type TXTBlockReader struct {
 	*LineReader
-	filetype  schema.FileType
+	classname string
 	firstline string
 	nextline  string
 }
 
-func NewTXTBlockReader(filename string, filetype schema.FileType, reader io.Reader) *TXTBlockReader {
+func NewTXTBlockReader(filename string, classname string, reader io.Reader) *TXTBlockReader {
 	return &TXTBlockReader{
 		LineReader: NewLineReader(filename, reader),
-		filetype:   filetype,
+		classname:  classname,
 	}
 }
 

schema/classinfo.go

@@ -140,10 +140,3 @@ func NewClassinfo(classaliasname, classsimplename, baseclassaliasname string, fi
 	}
 	return
 }
-
-var Relations = map[string]string{
-	"contains": "contain",
-	"contain":  "contain",
-	"depend":   "depend",
-	"topology": "topology",
-}

schema/edge.go

@@ -0,0 +1,10 @@
+package schema
+
+var Relations = map[string]string{
+	"contains": "contain",
+	"contain":  "contain",
+	"depend":   "depend",
+	"topology": "topology",
+}
+
+const EDGE = "EDGE"

schema/filetype.go

@@ -1,16 +0,0 @@
-package schema
-
-type FileType string
-
-const (
-	FT_LEVEL1 FileType = "level1"
-	FT_LEVEL2 FileType = "level2"
-	FT_LEVEL3 FileType = "level3"
-	FT_LEVEL4 FileType = "level4"
-	FT_LEVEL5 FileType = "level5"
-	FT_LEVEL6 FileType = "level6"
-	FT_LEVEL7 FileType = "level7"
-	FT_LEVEL8 FileType = "level8"
-	FT_MASTER FileType = "master"
-	FT_EDGE   FileType = "edge"
-)