rc queue limit

go.mod

@@ -4,10 +4,11 @@ go 1.20
 
 require (
 	git.wecise.com/wecise/odb-go v0.0.0-20250208123107-b502a8626316
-	git.wecise.com/wecise/util v0.0.0-20250211084319-911232a9300a
+	git.wecise.com/wecise/util v0.0.0-20250211091151-e5a2f521412f
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13
 	github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e
 	github.com/spf13/cast v1.7.0
+	github.com/wecisecode/util v0.0.0-20250211090829-552742b3dfbb
 	modernc.org/sqlite v1.30.1
 )
 

go.sum

@@ -1,7 +1,7 @@
 git.wecise.com/wecise/odb-go v0.0.0-20250208123107-b502a8626316 h1:4ljPCr2MUA9w3HGRiS4dwYuqKJAbuLBoaho70kO8D8I=
 git.wecise.com/wecise/odb-go v0.0.0-20250208123107-b502a8626316/go.mod h1:0/+7FWRDsMND6k6fgW836IXTYEd1vzOdBuIDEA2FnX8=
-git.wecise.com/wecise/util v0.0.0-20250211084319-911232a9300a h1:411X8V9jkrpo8Klj7gS/LuauhSv0UxiNSs1vt94WvSQ=
-git.wecise.com/wecise/util v0.0.0-20250211084319-911232a9300a/go.mod h1:2YXWE9m5mNgAu40zpYrL3woGz6S8CoHAW/CJeWXaIko=
+git.wecise.com/wecise/util v0.0.0-20250211091151-e5a2f521412f h1:NPco4igK5uBw7lTwFP4H4Yg06uEn6RPMh3xEbkgb1g4=
+git.wecise.com/wecise/util v0.0.0-20250211091151-e5a2f521412f/go.mod h1:9349QCkjPB2gs9a+kCzwHgMtOuf+KovY/2uRkLT4w3A=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
 github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
@@ -77,6 +77,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/wecisecode/util v0.0.0-20250211090829-552742b3dfbb h1:JNKgZsGGuGzbGG4i0ejVrmdu2mSRbIIfKOM9P6V66eI=
+github.com/wecisecode/util v0.0.0-20250211090829-552742b3dfbb/go.mod h1:JuKKMe2bEPXEDz3oFuCidZnW+0jdXWhUGFNPmkgsz78=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/etcd/api/v3 v3.5.8 h1:Zf44zJszoU7zRV0X/nStPenegNXoFDWcB/MwrJbA+L4=

graph/edgeinfo.go

@@ -6,8 +6,8 @@ import (
 
 	"git.wecise.com/wecise/cgimport/schema"
 	"git.wecise.com/wecise/util/deepcopy"
-	"git.wecise.com/wecise/util/merrs"
 	"github.com/spf13/cast"
+	"github.com/wecisecode/util/merrs"
 )
 
 type EdgeInfo struct {

graph/nodeid.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"regexp"
 
-	"git.wecise.com/wecise/util/merrs"
 	"github.com/spf13/cast"
+	"github.com/wecisecode/util/merrs"
 )
 
 func GetNodeId(classaliasname string, data map[string]any) (oid, suid string, err error) {

importer/datainfo.go

@@ -8,7 +8,7 @@ import (
 
 	"git.wecise.com/wecise/cgimport/schema"
 	"git.wecise.com/wecise/util/cmap"
-	"git.wecise.com/wecise/util/merrs"
+	"github.com/wecisecode/util/merrs"
 )
 
 type classdatainfo struct {

importer/importer.go

@@ -17,8 +17,8 @@ import (
 	"git.wecise.com/wecise/cgimport/reader"
 	"git.wecise.com/wecise/cgimport/schema"
 	"git.wecise.com/wecise/util/filewalker"
-	"git.wecise.com/wecise/util/merrs"
 	"git.wecise.com/wecise/util/rc"
+	"github.com/wecisecode/util/merrs"
 )
 
 var mcfg = odbc.Config
@@ -37,13 +37,14 @@ type Importer struct {
 }
 
 func ImportDir(datapath string, parallel int, reload bool) (totalfilescount, totalrecordscount int64, totalusetime time.Duration, filescount, recordscount int64, usetime time.Duration, err error) {
+	concurlimt := mcfg.GetInt("odbc.concurrent.limit", parallel*5)
 	importer := &Importer{
 		datapath:     datapath,
 		parallel:     parallel,
 		reload:       reload,
 		importstatus: NewCGIStatus(),
 		fileimportrc: rc.NewRoutinesController("", parallel),
-		odbcqueryrc:  rc.NewRoutinesController("", mcfg.GetInt("odbc.concurrent.limit", parallel*5)),
+		odbcqueryrc:  rc.NewRoutinesControllerLimit("", concurlimt, concurlimt*2),
 		odbcimporter: NewODBCImporter(),
 	}
 	return importer.Import()

importer/odbcimporter.go

@@ -12,9 +12,9 @@ import (
 	"git.wecise.com/wecise/odb-go/odb"
 	"git.wecise.com/wecise/util/cast"
 	"git.wecise.com/wecise/util/cmap"
-	"git.wecise.com/wecise/util/merrs"
 	"github.com/dgryski/go-farm"
 	"github.com/scylladb/go-set/strset"
+	"github.com/wecisecode/util/merrs"
 )
 
 type ODBCImporter struct {

reader/blockreader.go

@@ -5,7 +5,7 @@ import (
 	"path/filepath"
 
 	"git.wecise.com/wecise/cgimport/schema"
-	"git.wecise.com/wecise/util/merrs"
+	"github.com/wecisecode/util/merrs"
 )
 
 type BlockReader interface {

reader/csvreader.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 
 	"git.wecise.com/wecise/cgimport/schema"
-	"git.wecise.com/wecise/util/merrs"
 	"github.com/spf13/cast"
+	"github.com/wecisecode/util/merrs"
 )
 
 type CSVBlockReader struct {

sqlite/iter.go

@@ -5,7 +5,7 @@ import (
 	"database/sql"
 	"reflect"
 
-	"git.wecise.com/wecise/util/merrs"
+	"github.com/wecisecode/util/merrs"
 )
 
 type Row struct {

sqlite/sqldb.go

@@ -13,10 +13,10 @@ import (
 
 	"git.wecise.com/wecise/cgimport/sqlite/schema"
 	"git.wecise.com/wecise/util/cmap"
-	"git.wecise.com/wecise/util/merrs"
 	"git.wecise.com/wecise/util/msgpack"
 	"git.wecise.com/wecise/util/rc"
 	"github.com/spf13/cast"
+	"github.com/wecisecode/util/merrs"
 	"modernc.org/sqlite"
 	sqlite3 "modernc.org/sqlite/lib"
 )