cgimport/main.go

package main

import (
	"fmt"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"git.wecise.com/wecise/cgimport/cgf"
	"git.wecise.com/wecise/util/cfg"
	"git.wecise.com/wecise/util/filewalker"
	"git.wecise.com/wecise/util/rc"
)

// 获取配置信息
//
//	默认配置：
//		当前工作目录下 与应用同名的 .conf 文件
//		环境变量
//		命令行参数
var mcfg = cfg.MConfig()

func main() {
	st := time.Now()
	filescount := int64(0)
	recordscount := int64(0)

	// 文件目录
	datapath := mcfg.GetStrings("datapath", "data")
	// 并发数
	parallel := mcfg.GetInt("parallel", 10)

	// 遍历文件目录
	var cgirc = rc.NewRoutinesController("", parallel)
	var wg sync.WaitGroup
	var err error
	fw, e := filewalker.NewFileWalker(datapath, ".*")
	if e != nil {
		panic(e)
	}
	e = fw.List(func(basedir string, fpath string) bool {
		if err != nil {
			return false
		}
		filename := filepath.Join(basedir, fpath)
		wg.Add(1)
		cgirc.ConcurCall(1,
			func() {
				defer wg.Done()
				records, e := cgf.Import(filename)
				if e != nil {
					err = e
					return
				}
				atomic.AddInt64(&filescount, 1)
				atomic.AddInt64(&recordscount, int64(records))
			},
		)
		return true
	})
	wg.Wait()
	if e != nil {
		panic(e)
	}
	if err != nil {
		panic(err)
	}
	fmt.Println("total import", filescount, "files", recordscount, "records", "in", time.Since(st))
}