mql_run.go 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345
  1. package odbcmql
  2. import (
  3. "bufio"
  4. "context"
  5. "encoding/json"
  6. "fmt"
  7. "math/rand"
  8. "os"
  9. "path/filepath"
  10. "regexp"
  11. "runtime"
  12. "strconv"
  13. "strings"
  14. "sync"
  15. "sync/atomic"
  16. "testing"
  17. "time"
  18. odb "git.wecise.com/wecise/odb-go/odb"
  19. "git.wecise.com/wecise/odb-go/odb/mql/qstru"
  20. "gitee.com/wecisecode/util/cast"
  21. mcfg "gitee.com/wecisecode/util/cfg"
  22. "gitee.com/wecisecode/util/filewalker"
  23. "gitee.com/wecisecode/util/pqc"
  24. "gitee.com/wecisecode/util/set/strset"
  25. "gitee.com/wecisecode/util/spliter"
  26. "github.com/stretchr/testify/assert"
  27. )
  28. var ODBC odb.Client
  29. var ODBError error
  30. var debug bool
  31. // var checkCassSchema bool
  32. var default_keyspace = `oktest`
  33. var default_odbpaths = `127.0.0.1:11001`
  34. var ksnative = default_keyspace + "_native"
  35. func init() {
  36. logprefix := config.GetString("log.prefix", "")
  37. if logprefix != "" {
  38. logprefix += " "
  39. }
  40. logger.SetFormat(logprefix+"yyyy-MM-dd HH:mm:ss.SSSSSS [pid] [level] msg", "\n")
  41. odbpaths := strset.New(strings.Split(mcfg.CommandArgs.GetString("odb",
  42. strings.Join(mcfg.Environs.GetStrings("ODBPATH",
  43. config.GetStrings("odbc.odbpath", default_odbpaths)...), ",")), ",")...).List()
  44. keyspace := mcfg.CommandArgs.GetString("keyspace",
  45. mcfg.Environs.GetString("KEYSPACE", config.GetString("odbc.keyspace", default_keyspace)))
  46. cassdc := mcfg.CommandArgs.GetString("cassandra.dc",
  47. mcfg.Environs.GetString("CASSANDRADC", config.GetString("cassandra.dc", "")))
  48. if cassdc == "" || cassdc == "dc1" {
  49. ksnative = keyspace + "_native"
  50. } else {
  51. ksnative = keyspace + "_" + cassdc + "_native"
  52. }
  53. ksnative = mcfg.CommandArgs.GetString("ksnative",
  54. mcfg.Environs.GetString("KSNATIVE", config.GetString("odbc.ksnative", ksnative)))
  55. debug = mcfg.CommandArgs.GetBool("debug", false)
  56. // checkCassSchema = mcfg.CommandArgs.GetBool("ccs", true)
  57. ODBC, ODBError = odb.NewClient(&odb.Config{
  58. Keyspace: keyspace,
  59. Hosts: odbpaths,
  60. Debug: debug,
  61. })
  62. }
  63. func Run(t *testing.T) {
  64. if mcfg.CommandArgs.GetString("keyspace?") == "keyspace?" {
  65. if ODBC != nil {
  66. fmt.Println(ODBC.Config().Keyspace)
  67. os.Exit(0)
  68. }
  69. os.Exit(0)
  70. }
  71. if mcfg.CommandArgs.GetString("odbpath?") == "odbpath?" {
  72. if ODBC != nil {
  73. fmt.Println(ODBC.Config().Hosts)
  74. os.Exit(0)
  75. }
  76. os.Exit(0)
  77. }
  78. if mcfg.CommandArgs.GetString("odbver?") == "odbver?" {
  79. if ODBC != nil {
  80. fmt.Println(ODBC.Versline())
  81. os.Exit(0)
  82. }
  83. os.Exit(0)
  84. }
  85. spath := strings.Split(config.GetString("test.mql.path"), " ")
  86. if len(os.Args) > 1 {
  87. if len(os.Args) > 2 && os.Args[1] == "-" {
  88. s := strings.Join(os.Args[2:], " ")
  89. spath = []string{"/tmp/1.mql"}
  90. os.WriteFile(spath[0], []byte(s+"/**output()**/"), os.ModePerm)
  91. } else {
  92. for i := 1; i < len(os.Args); i++ {
  93. ap := os.Args[i]
  94. if strings.HasPrefix(ap, "-test.") {
  95. break
  96. }
  97. if strings.Contains(ap, "=") {
  98. continue
  99. }
  100. if (ap == "-" || ap == "--") && i+1 < len(os.Args) {
  101. s := strings.Join(os.Args[i+1:], " ")
  102. spath = []string{"/tmp/1.mql"}
  103. os.WriteFile(spath[0], []byte(s+"\n/**output()**/"), os.ModePerm)
  104. break
  105. }
  106. spath = append(spath, ap)
  107. }
  108. }
  109. }
  110. if len(spath) == 0 || len(spath) == 1 && len(spath[0]) == 0 {
  111. fmt.Print(`循环遍历执行指定路径下的 mql 文件
  112. usage: mql path [options] -- mqlstatement
  113. path 指定mql文件所在路径,通配符 ** 表示任意字符,* 表示除分隔符以外的任意字符, . 表示递归当前目录下的所有子目录
  114. 为避免 shell 自动将 * 转换为文件名列表,可以将指定的 path 用引号包含
  115. mqlstatement 指定要执行的 mql 语句,支持多条语句,语句之间用分号 ; 分隔
  116. options:
  117. match=^\\d+.* 文件名匹配正则表达式,默认为所有以数字开头命名的文件
  118. odb=` + default_odbpaths + ` 指定odbserver路径,默认通过环境变量ODBPATH或通过ETCD相关配置获取
  119. keyspace=` + default_keyspace + ` 指定keyspace,默认通过环境变量KEYSPACE获取
  120. ksnative=` + default_keyspace + `_native 指定native keyspace,默认通过环境变量KSNATIVE获取
  121. debug=true 开启调试模式,输出更多信息
  122. 环境变量需求:
  123. KEYSPACE=` + default_keyspace + `
  124. KSNATIVE=` + default_keyspace + `_native
  125. ODBPATH=` + default_odbpaths + `
  126. ETCDPATH=127.0.0.1:2379
  127. ETCDUSER=
  128. ETCDPASS=
  129. CASSANDRAPATH=127.0.0.1
  130. CASSANDRALOOKUP=false
  131. NATSPATH=nats://user:N5JbKeT1C3uOUh317OVXjg==@127.0.0.1:4222
  132. LANG=zh_CN.utf8
  133. LC_ALL=zh_CN.utf8
  134. 可通过mql.conf配置运行参数:
  135. [odbc]
  136. ;指定odbserver路径
  137. odbpath=` + default_odbpaths + `
  138. ;指定keyspace
  139. keyspace=` + default_keyspace + `
  140. ;指定native keyspace
  141. ksnative=` + default_keyspace + `_native
  142. mql语句扩展说明:
  143. mql语句中的内容替换:
  144. 随机字符替换:
  145. mql语句中的 '...N Bytes...' 会被替换为 N 个随机可见字符,N 为自然数
  146. 变量替换:
  147. mql语句中 '{%<f>,<varname>}' 形式的内容会被替换为,以 %<f> 作为格式化标记的 <varname> 变量的内容
  148. 目前支持的 <varname> 包括:
  149. keyspace 当前指定的 Keyspace
  150. ksnative Native Keyspace
  151. mqli 当前mql语句的循环值,参考 loop(N)
  152. filei 当前文件的循环值,参考 scope(file) loop(N)
  153. diri 当前目录的循环值,参考 scope(dir) loop(N)
  154. topi 整个执行进程的循环值,参考 scope(top) loop(N)
  155. mqlcount 当前mql语句的总计数
  156. filecount 当前文件的总计数
  157. dircount 当前目录的总计数
  158. topcount 整个执行进程的总计数
  159. mql语句中第一个 /** 到 最后一个 **/ 之间的内容可以标注执行若干预定义动作,多个动作标注用空格分隔
  160. 如 /** output() sleep(1) **/ 表示忽略当前语句执行过程中的任何报错,执行完成后等待一秒后再继续执行下一语句
  161. 目前支持的预定义动作如下:
  162. skip() 跳过当前语句,以及其它的所有预定义动作
  163. 可用于暂时屏蔽一条语句的执行
  164. params(V) 用于提供 prepare 语句中 ? 的对应值,V 为 JSON 格式编码的数组
  165. 对应值中不能有包含逗号和右括号,如果需要逗号用 \u002c 代替,右括号用 \u0029 代替
  166. subscribe(S) 订阅指定主题的消息通知 S,输出收到的通知信息,S 为要订阅主题的的名称字符串,同一主题只会被订阅一次
  167. unsubscribe(S) 取消订阅指定主题的消息通知 S,S 为要订阅主题的名称字符串
  168. fork(G) 新建并行线程分支,执行当前语句的同时,继续执行后续语句,G 为线程组名,配合 wait 使用
  169. wait(G) 执行当前语句前,等待之前创建的分支结束,G 为线程组名,对应由 fork 创建的所有同名线程分支,G 为空表示所有
  170. beforerun(F,...) 执行当前语句前,执行一系列预定义函数,以通过一些简单逻辑处理产生模拟数据,目前支持的函数包括
  171. set(S,V) 设置变量,S 为变量名,V为变量值
  172. add(S,V,u) 累加变量,S 为变量名,V为增量值,u为可选的变量单位,如时间段单位 second,minute,hour
  173. mod(S,V) 变量取模,相当于 S = S % V
  174. case(C,F) 根据条件 C 执行函数 F
  175. ------------- 针对错误处理的相关动作
  176. retry(N) 出错重试 N 次,N为自然数,默认为 0,
  177. onerror(O,E,...) retry后仍然有错时的处理方法,E为特定的错误信息关键字,不指定则为任意错误,O为处理方法,包括:
  178. break 中断当前语句的循环,继续顺序执行其它语句,没有循环时与pass无区别
  179. continue 忽略报错,继续执行,在循环中时,继续下一轮循环
  180. must 报错才正常,否则中断执行,报告期待错误信息不符,退出当前进程
  181. exit 停止执行,退出当前进程,此为默认处理方式
  182. noerrinfo() 存在 retry 或 onerror 处理时,不输出错误信息
  183. ------------- 以上的预定义动作只对当前语句起作用,下面的预定义动作可以指定其作用范围
  184. scope(S) 相关动作的作用范围 S 包括 top,dir,file,mql,默认作用范围为 mql 当前语句
  185. 如:scope(file) parallel(3) loop(5) 并发循环执行本文件五次,最大并行数限制为3
  186. qmeta(O) mql执行时设置特定的选项,O为json对象
  187. timeout(D) mql执行超时设置,D 为时间段,默认为一分钟
  188. 时间段 D 支持单位 d(天),h(时),m(分),s(秒),ms(毫秒),us(微秒),ns(纳秒),默认为毫秒
  189. loop(N) 循环执行 N 次,N 为正整数,默认执行一次
  190. 执行前会替换 mql 语句及 params 参数中的循环次数标记
  191. loopfrom(N) 设置循环执行计数起始值 N,默认为 1,该值仅影响替换 mql 语句及 params 参数中的循环次数标记
  192. loopstep(N) 设置循环执行计数步长值 N,默认为 1,该值仅影响替换 mql 语句及 params 参数中的循环次数标记
  193. parallel(N) 并发执行,N 为最大并行数,不指定 N 表示不限制并行数
  194. set(S,V) 设置变量,S 为变量名,V为变量值
  195. ------------- 以上为执行语句前的预定义动作,下面是执行完成后的预定义动作
  196. sleep(D) 执行完成后等待时间段 D 后再继续执行下一语句
  197. output() 输出执行结果
  198. outputcount() 输出执行结果记录条数
  199. metainfo() 输出执行结果相关的元信息
  200. schema(C) 执行 schema 命令检查指定类 C 是否存在
  201. count(N) 检查返回结果中的数据记录数是否为 N,N 为自然数
  202. equal(N,F,V) 判断返回结果中第 N 条数据的字段 F 的值是否为 V
  203. match(Kn,Mn) 检查返回结果中字段 Kn 值为 Mn 的记录是否存在,参数 Kn,Mn 可以有多个,需要成对出现
  204. matchcount(Kn,Mn,N) 检查返回结果中字段 Kn 值为 Mn 的记录数是否为 N,参数 Kn,Mn 可以有多个,需要成对出现,N为自然数
  205. 例:循环遍历执行当前路径下的所有 mql 文件
  206. ./mql .
  207. `)
  208. os.Exit(0)
  209. return
  210. }
  211. // 确定MODB连接
  212. if !assert.Nil(t, ODBError) {
  213. return
  214. }
  215. logger.Info("odbpath :", ODBC.Config().Hosts, ODBC.Config().Port)
  216. logger.Info("keyspace :", ODBC.Config().Keyspace)
  217. logger.Info("version :", ODBC.Versline())
  218. logger.Info("ksnative :", ksnative)
  219. logger.Info("debug :", debug)
  220. logger.Info("spath :", spath)
  221. // if checkCassSchema && !cass.CheckCassandraSchema(t) {
  222. // assert.True(t, false, "cassandra schemda is different")
  223. // return
  224. // }
  225. // 文件名以数字开头
  226. fnmatch := mcfg.CommandArgs.GetString("match", `^\d+.*`)
  227. fw, err := filewalker.NewFileWalker(spath, fnmatch) // orderby: dirfirst, filefirst, fullpath
  228. if !assert.Nil(t, err, err) {
  229. return
  230. }
  231. logger.Info("walkdir: ", fw.WalkDir)
  232. logger.Info("pathmatch: ", fw.RePath)
  233. logger.Info("filematch: ", fw.ReFile)
  234. // test
  235. fns := []string{}
  236. fw.List(func(basedir, fpath string) bool { fns = append(fns, filepath.Join(basedir, fpath)); return true })
  237. logger.Info(fmt.Sprint("fns:", "\n", strings.Join(fns, "\n")))
  238. if len(fns) == 0 {
  239. logger.Info("没有找到 MQL 文件")
  240. return
  241. }
  242. logger.Info("共找到", len(fns), "个文件")
  243. // 顺序读取当前目录下文件名为数字开头的文件,执行其中的mql
  244. (&MQLTest{}).Run(t, fw)
  245. // 等待延迟显示信息输出
  246. // time.Sleep(1 * time.Second)
  247. logger.Info("关闭与服务器的连接")
  248. ODBC.Close()
  249. // 等待正常结束,只是为了发现隐含的问题
  250. // for n := 0; runtime.NumGoroutine() > 60 && n < 10; n++ {
  251. // fmt.Println("剩余协程数:", runtime.NumGoroutine())
  252. // time.Sleep(1000 * time.Millisecond)
  253. // }
  254. logger.Info("剩余协程数:", runtime.NumGoroutine())
  255. }
  256. // action(a1,a2...)
  257. // --注释后的大括号之间的内容 -- {}
  258. // mql query之后执行的动作
  259. var reactions = regexp.MustCompile(`(\w+)\s*\(([^\)]*)\)`)
  260. // 替换为随机可见字符
  261. var rereplace_nbytes = regexp.MustCompile(`\.\.\.\s*(\d+)\s*[Bb][Yy][Tt][Ee]s?\s*\.\.\.`)
  262. var reactioneexprs = regexp.MustCompile(`(?s)\/\*\*(?:\s*(\w+\s*\(\s*.*\)\s*)*)*\*\*\/`)
  263. var reactioneexprs_2 = regexp.MustCompile(`(?s)(?:(?:^|\n)\-\-\s*\{(.*)\}\s*)+$`)
  264. var reactioneexprs_3 = regexp.MustCompile(`(?s)(?:^\-\-\s*\{(.*)\}\s*)+\n`)
  265. var commentexprs = regexp.MustCompile(`(?s)\/\*(?:[^\*]|\*+[^\*\/])*\*+\/`)
  266. var commentexprs_2 = regexp.MustCompile(`(?ms)(?:^|\n)\-\-[^\n]*(?:\n|$)`)
  267. var commentexprs_3 = regexp.MustCompile(`(?ms)(?:^|\n)//[^\n]*(?:\n|$)`)
  268. type MQLTest struct {
  269. t *testing.T
  270. fw *filewalker.FileWalker
  271. scopevars *ScopeVars
  272. }
  273. type ScopeVars struct {
  274. sync.RWMutex
  275. top *Variables
  276. dir map[string]*Variables
  277. file map[string]*Variables
  278. mql map[string]*Variables
  279. }
  280. type Variables struct {
  281. vars map[string]interface{}
  282. loop_count int
  283. loop_from int
  284. loop_step int
  285. timeout time.Duration
  286. qmeta odb.QueryMeta
  287. parallel_max int
  288. }
  289. type CurrentVars struct {
  290. loop_i int
  291. ch_parallel_count chan<- int
  292. mqlcount int32
  293. sleeptime time.Duration
  294. totalusetime time.Duration
  295. maxusetime time.Duration
  296. minusetime time.Duration
  297. }
  298. type GlobalVars struct {
  299. sync.RWMutex
  300. *CurrentVars
  301. wg_wait_fork_routine map[string]*sync.WaitGroup
  302. ch_wait_mql_done map[string]chan bool
  303. }
  304. func (mt *MQLTest) Run(t *testing.T, fw *filewalker.FileWalker) (ok bool) {
  305. mt.t = t
  306. mt.fw = fw
  307. global := &GlobalVars{
  308. CurrentVars: &CurrentVars{},
  309. wg_wait_fork_routine: make(map[string]*sync.WaitGroup),
  310. ch_wait_mql_done: make(map[string]chan bool),
  311. }
  312. mt.scopevars = &ScopeVars{
  313. top: &Variables{
  314. vars: map[string]interface{}{},
  315. loop_count: 1,
  316. loop_from: 1,
  317. loop_step: 1},
  318. dir: map[string]*Variables{},
  319. file: map[string]*Variables{},
  320. mql: map[string]*Variables{}}
  321. ctx, cancel := context.WithCancel(context.Background())
  322. defer cancel()
  323. var wg sync.WaitGroup
  324. st := time.Now()
  325. loop_i := 0
  326. parallel_queue := pqc.NewQueue[any](0)
  327. mqlcount := int32(0)
  328. for {
  329. mt.scopevars.Lock()
  330. ok := loop_i < mt.scopevars.top.loop_count
  331. mt.scopevars.Unlock()
  332. if !ok {
  333. break
  334. }
  335. loop_i++
  336. //
  337. ch_parallel_count := make(chan int)
  338. topvars := &CurrentVars{
  339. loop_i: loop_i,
  340. ch_parallel_count: ch_parallel_count,
  341. }
  342. //
  343. ok_chan := make(chan bool, 1)
  344. parallel_chan := make(chan bool, 1)
  345. parallel := false
  346. parallelcount := 0
  347. done := false
  348. wg.Add(1)
  349. go func() {
  350. defer func() {
  351. atomic.AddInt32(&mqlcount, topvars.mqlcount)
  352. wg.Done()
  353. }()
  354. ch_ok := make(chan bool)
  355. go func() {
  356. for {
  357. select {
  358. case <-ch_parallel_count:
  359. if !done && !parallel {
  360. parallel = true
  361. // 加入并发控制队列
  362. if parallelcount > 0 {
  363. if parallelcount > parallel_queue.Size() {
  364. parallel_queue.Growth(parallelcount)
  365. }
  366. parallel_queue.Push(1)
  367. }
  368. parallel_chan <- true
  369. }
  370. case ok := <-ch_ok:
  371. ok_chan <- ok
  372. if parallel {
  373. if parallelcount > 0 {
  374. // 从并发控制队列中移除
  375. parallel_queue.Pop()
  376. }
  377. } else {
  378. done = true
  379. }
  380. return
  381. }
  382. }
  383. }()
  384. ok := mt.RunAll(t, ctx,
  385. global,
  386. topvars)
  387. ch_ok <- ok
  388. if !ok {
  389. cancel() // 并发测试,有一个线程出错,就全停
  390. return
  391. }
  392. }()
  393. success := true
  394. select {
  395. case success = <-ok_chan: // 非并发,等待完成
  396. case <-parallel_chan: // 并发,执行继续下一次
  397. logger.Info("第", topvars.loop_i, "次并发执行继续")
  398. }
  399. if !success {
  400. return false // 失败,不等,直接返回
  401. }
  402. }
  403. wg.Wait()
  404. mt.scopevars.RLock()
  405. loop_count := mt.scopevars.top.loop_count
  406. mt.scopevars.RUnlock()
  407. ut := time.Since(st)
  408. aut := time.Duration(0)
  409. if mqlcount > 0 {
  410. aut = global.totalusetime / time.Duration(mqlcount)
  411. }
  412. logger.Info("完成 ", loop_count, " 次执行,共", mqlcount, "次 MQL 请求,耗时", ut, "单条响应时间", global.minusetime, "~", global.maxusetime, "/", aut, "平均每秒吞吐量", (int64(mqlcount)*int64(time.Second))/int64(ut))
  413. return
  414. }
  415. func (mt *MQLTest) RunAll(t *testing.T, ctx context.Context, global *GlobalVars, topvars *CurrentVars) bool {
  416. logger.Info("开始第", topvars.loop_i, "次执行")
  417. st := time.Now()
  418. // 读取文件列表
  419. listdirs := []string{}
  420. dirfiles := map[string][]string{}
  421. err := mt.fw.List(func(basedir, filename string) bool {
  422. if dirfiles[basedir] == nil {
  423. listdirs = append(listdirs, basedir)
  424. }
  425. dirfiles[basedir] = append(dirfiles[basedir], filename)
  426. return true
  427. })
  428. if !assert.Nil(t, err, err) {
  429. return false
  430. }
  431. // 执行所有目录
  432. for _, basedir := range listdirs {
  433. // 循环执行目录下所有文件
  434. mt.scopevars.Lock()
  435. if mt.scopevars.dir[basedir] == nil {
  436. mt.scopevars.dir[basedir] = &Variables{
  437. vars: map[string]interface{}{},
  438. loop_count: 1,
  439. loop_from: 1,
  440. loop_step: 1}
  441. }
  442. mt.scopevars.Unlock()
  443. var wg sync.WaitGroup
  444. st := time.Now()
  445. loop_i := 0
  446. parallel_queue := pqc.NewQueue[any](0)
  447. mqlcount := int32(0)
  448. for {
  449. mt.scopevars.Lock()
  450. ok := loop_i < mt.scopevars.dir[basedir].loop_count
  451. mt.scopevars.Unlock()
  452. if !ok {
  453. break
  454. }
  455. loop_i++
  456. //
  457. ch_parallel_count := make(chan int)
  458. dirvars := &CurrentVars{
  459. loop_i: loop_i,
  460. ch_parallel_count: ch_parallel_count,
  461. }
  462. //
  463. ok_chan := make(chan bool, 1)
  464. parallel_chan := make(chan bool, 1)
  465. parallel := false
  466. parallelcount := 0
  467. done := false
  468. wg.Add(1)
  469. go func(basedir string) {
  470. defer func() {
  471. atomic.AddInt32(&mqlcount, dirvars.mqlcount)
  472. wg.Done()
  473. }()
  474. ch_ok := make(chan bool)
  475. go func() {
  476. for {
  477. select {
  478. case <-ch_parallel_count:
  479. if !done && !parallel {
  480. parallel = true
  481. // 加入并发控制队列
  482. if parallelcount > 0 {
  483. if parallelcount > parallel_queue.Size() {
  484. parallel_queue.Growth(parallelcount)
  485. }
  486. parallel_queue.Push(1)
  487. }
  488. parallel_chan <- true
  489. }
  490. case ok := <-ch_ok:
  491. ok_chan <- ok
  492. if parallel {
  493. if parallelcount > 0 {
  494. // 从并发控制队列中移除
  495. parallel_queue.Pop()
  496. }
  497. } else {
  498. done = true
  499. }
  500. return
  501. }
  502. }
  503. }()
  504. // logger.Info("dir", basedir, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i), "次执行开始")
  505. files := dirfiles[basedir]
  506. ch_ok <- mt.RunDir(t, ctx,
  507. global,
  508. topvars,
  509. dirvars,
  510. basedir, files)
  511. // logger.Info("dir", basedir, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i), "次执行结束")
  512. }(basedir)
  513. success := true
  514. select {
  515. case success = <-ok_chan: // 非并发,等待完成
  516. logger.Info("dir", basedir, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i), "次顺序执行完成")
  517. case <-parallel_chan: // 并发,执行继续下一次
  518. logger.Info("dir", basedir, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i), "次并发执行继续")
  519. }
  520. if !success {
  521. return false
  522. }
  523. }
  524. wg.Wait()
  525. mt.scopevars.RLock()
  526. loop_count := mt.scopevars.dir[basedir].loop_count
  527. mt.scopevars.RUnlock()
  528. if loop_count > 1 {
  529. ut := time.Since(st)
  530. logger.Info(fmt.Sprint("dir ", basedir, " loop ", loop_count, " times, run ", mqlcount, " mqls, usetime ", ut))
  531. }
  532. }
  533. logger.Info("完成第", topvars.loop_i, "次执行,共", topvars.mqlcount, "次 MQL 请求,耗时", time.Since(st))
  534. return true
  535. }
  536. func (mt *MQLTest) RunDir(t *testing.T, ctx context.Context,
  537. global *GlobalVars,
  538. topvars *CurrentVars,
  539. dirvars *CurrentVars,
  540. basedir string, files []string) bool {
  541. for _, filename := range files {
  542. ok := mt.RunFile(t, ctx, global, topvars, dirvars, basedir, filename)
  543. if !ok {
  544. return false
  545. }
  546. }
  547. return true
  548. }
  549. func (mt *MQLTest) RunFile(t *testing.T, ctx context.Context,
  550. global *GlobalVars,
  551. topvars *CurrentVars,
  552. dirvars *CurrentVars,
  553. basedir, filename string) bool {
  554. // 读取文件内容
  555. ffpath := filepath.Join(basedir, filename)
  556. bs, err := os.ReadFile(ffpath)
  557. if !assert.Nil(t, err, err) {
  558. return false
  559. }
  560. // mql语句切分
  561. mqgr := NewMQLGroupRequest()
  562. var multilines *MQLRequest
  563. mqs := spliter.NewMQLSpliter(bufio.NewReader(strings.NewReader(string(bs))))
  564. for {
  565. mql, fromline, toline, fromchar, tochar, hasnext, _ := mqs.NextMQL()
  566. if !hasnext {
  567. break
  568. }
  569. if multilines != nil {
  570. if strings.TrimSpace(mql) == "multilines end" {
  571. e := mqgr.Append(multilines)
  572. if !assert.Nil(t, e, e) {
  573. return false
  574. }
  575. multilines = nil
  576. } else if multilines.OriginQueryString == "" {
  577. multilines.OriginQueryString = mql
  578. multilines.Fromline = fromline
  579. multilines.Toline = toline
  580. multilines.Fromchar = fromchar
  581. multilines.Tochar = tochar
  582. } else {
  583. multilines.OriginQueryString += ";" + mql
  584. multilines.Toline = toline
  585. multilines.Tochar = tochar
  586. }
  587. } else if strings.TrimSpace(mql) == "multilines begin" {
  588. multilines = &MQLRequest{FilePath: ffpath}
  589. } else {
  590. mqr := &MQLRequest{OriginQueryString: mql, FilePath: ffpath, Fromline: fromline, Toline: toline, Fromchar: fromchar, Tochar: tochar}
  591. e := mqgr.Append(mqr)
  592. if !assert.Nil(t, e, e) {
  593. return false
  594. }
  595. }
  596. }
  597. // mqls := spliter.MQLSplit(string(bs))
  598. mt.scopevars.Lock()
  599. if mt.scopevars.file[ffpath] == nil {
  600. mt.scopevars.file[ffpath] = &Variables{
  601. vars: map[string]interface{}{},
  602. loop_count: 1,
  603. loop_from: 1,
  604. loop_step: 1}
  605. }
  606. mt.scopevars.Unlock()
  607. var wg sync.WaitGroup
  608. st := time.Now()
  609. loop_i := 0
  610. parallel_queue := pqc.NewQueue[any](0)
  611. mqlcount := int32(0)
  612. for {
  613. mt.scopevars.Lock()
  614. ok := loop_i < mt.scopevars.file[ffpath].loop_count
  615. mt.scopevars.Unlock()
  616. if !ok {
  617. break
  618. }
  619. loop_i++
  620. //
  621. ch_parallel_count := make(chan int)
  622. filevars := &CurrentVars{
  623. loop_i: loop_i,
  624. ch_parallel_count: ch_parallel_count,
  625. }
  626. //
  627. ok_chan := make(chan bool, 1)
  628. parallel_chan := make(chan bool, 1)
  629. parallel := false
  630. parallelcount := 0
  631. done := false
  632. wg.Add(1)
  633. go func() {
  634. defer func() {
  635. atomic.AddInt32(&mqlcount, filevars.mqlcount)
  636. wg.Done()
  637. }()
  638. ch_ok := make(chan bool)
  639. go func() {
  640. for {
  641. select {
  642. case <-ch_parallel_count:
  643. if !done && !parallel {
  644. parallel = true
  645. // 加入并发控制队列
  646. if parallelcount > 0 {
  647. if parallelcount > parallel_queue.Size() {
  648. parallel_queue.Growth(parallelcount)
  649. }
  650. parallel_queue.Push(1)
  651. }
  652. parallel_chan <- true
  653. }
  654. case ok := <-ch_ok:
  655. ok_chan <- ok
  656. if parallel {
  657. if parallelcount > 0 {
  658. // 从并发控制队列中移除
  659. parallel_queue.Pop()
  660. }
  661. } else {
  662. done = true
  663. }
  664. return
  665. }
  666. }
  667. }()
  668. // logger.Info("file", ffpath, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i), "次执行开始")
  669. ch_ok <- mt.RunMQLGroup(t, ctx,
  670. global,
  671. topvars,
  672. dirvars,
  673. filevars,
  674. basedir, ffpath, mqgr)
  675. // logger.Info("file", ffpath, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i), "次执行结束")
  676. }()
  677. success := true
  678. select {
  679. case success = <-ok_chan: // 非并发,等待完成
  680. logger.Info("file", ffpath, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i), "次顺序执行完成")
  681. case <-parallel_chan: // 并发,执行继续下一次
  682. logger.Info("file", ffpath, "第", fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i), "次并发执行继续")
  683. }
  684. if !success {
  685. return false
  686. }
  687. }
  688. wg.Wait()
  689. mt.scopevars.RLock()
  690. loop_count := mt.scopevars.file[ffpath].loop_count
  691. mt.scopevars.RUnlock()
  692. if loop_count > 1 {
  693. ut := time.Since(st)
  694. sn := fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i)
  695. logger.Info(fmt.Sprint("file ", ffpath+"/"+sn, " loop ", loop_count, " times, run ", mqlcount, " mqls, usetime ", ut))
  696. }
  697. return true
  698. }
  699. func (mt *MQLTest) RunMQLGroup(t *testing.T, ctx context.Context,
  700. global *GlobalVars,
  701. topvars *CurrentVars,
  702. dirvars *CurrentVars,
  703. filevars *CurrentVars,
  704. basedir, ffpath string, mqgr *MQLGroupRequest) (pass bool) {
  705. pass = true
  706. var wg sync.WaitGroup
  707. for _, mqs := range mqgr.mqrs {
  708. wg.Add(1)
  709. go func(mqs []*MQLRequest) {
  710. defer wg.Done()
  711. if len(mqs) == 0 {
  712. return
  713. }
  714. if mqs[0].StaticActions.ForkName != nil {
  715. forkname := *mqs[0].StaticActions.ForkName
  716. global.Lock()
  717. wg := global.wg_wait_fork_routine[forkname]
  718. if wg == nil {
  719. wg = &sync.WaitGroup{}
  720. global.wg_wait_fork_routine[forkname] = wg
  721. }
  722. global.Unlock()
  723. wg.Add(1)
  724. defer wg.Done()
  725. }
  726. ok := mt.RunMQLs(t, ctx, global, topvars, dirvars, filevars, basedir, ffpath, mqs)
  727. if !ok {
  728. pass = false
  729. }
  730. }(mqs)
  731. }
  732. wg.Wait()
  733. return
  734. }
  735. func (mt *MQLTest) RunMQLs(t *testing.T, ctx context.Context,
  736. global *GlobalVars,
  737. topvars *CurrentVars,
  738. dirvars *CurrentVars,
  739. filevars *CurrentVars,
  740. basedir, ffpath string, mqrs []*MQLRequest) bool {
  741. for _, mqr := range mqrs {
  742. mqlkey := mqr.Key
  743. mqlstr := mqr.OriginQueryString
  744. if mqlstr == "" {
  745. continue
  746. }
  747. staticactions := mqr.StaticActions
  748. // 设置执行过程中的控制参数
  749. mt.InitScopeVars(basedir, ffpath, mqlkey, staticactions)
  750. ch_test_run_one_mql_result := make(chan bool)
  751. go func() {
  752. mqrinst := fmt.Sprint(mqlkey, "/", topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i)
  753. global.Lock()
  754. mqrdone := global.ch_wait_mql_done[mqrinst]
  755. if mqrdone == nil {
  756. mqrdone = make(chan bool)
  757. global.ch_wait_mql_done[mqrinst] = mqrdone
  758. }
  759. var waitmqrdone chan bool
  760. if mqr.WaitMQLRequest != nil {
  761. waitmqrkey := mqr.WaitMQLRequest.Key
  762. waitmqrinst := fmt.Sprint(waitmqrkey, "/", topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i)
  763. waitmqrdone = global.ch_wait_mql_done[waitmqrinst]
  764. if waitmqrdone == nil {
  765. waitmqrdone = make(chan bool)
  766. global.ch_wait_mql_done[waitmqrinst] = waitmqrdone
  767. }
  768. }
  769. global.Unlock()
  770. var ret bool
  771. defer func() { mqrdone <- ret }()
  772. if waitmqrdone != nil {
  773. v := <-waitmqrdone
  774. waitmqrdone <- v
  775. if !v {
  776. // 依赖MQR失败
  777. ch_test_run_one_mql_result <- false
  778. return
  779. }
  780. }
  781. ret = mt.RunMQR(t, ctx, global, topvars, dirvars, filevars, basedir, ffpath, mqr)
  782. ch_test_run_one_mql_result <- ret
  783. }()
  784. ret := <-ch_test_run_one_mql_result
  785. if !ret {
  786. return ret
  787. }
  788. // continue
  789. }
  790. return true
  791. }
  792. func (mt *MQLTest) RunMQR(t *testing.T, ctx context.Context,
  793. global *GlobalVars,
  794. topvars *CurrentVars,
  795. dirvars *CurrentVars,
  796. filevars *CurrentVars,
  797. basedir, ffpath string, mqr *MQLRequest) bool {
  798. mqlkey := mqr.Key
  799. staticactionexprs := mqr.StaticActionExprs
  800. staticactions := mqr.StaticActions
  801. return t.Run(mqlkey, func(t *testing.T) {
  802. var wg sync.WaitGroup
  803. st := time.Now()
  804. loop_i := 0
  805. parallel_queue := pqc.NewQueue[any](0)
  806. mqlcount := int32(0)
  807. for {
  808. mt.scopevars.Lock()
  809. ok := loop_i < mt.scopevars.mql[mqlkey].loop_count
  810. mt.scopevars.Unlock()
  811. if !ok {
  812. break
  813. }
  814. loop_i++
  815. //
  816. ch_parallel_count := make(chan int)
  817. mqlvars := &CurrentVars{
  818. loop_i: loop_i,
  819. ch_parallel_count: ch_parallel_count,
  820. }
  821. // 运行实例
  822. mqlsn := fmt.Sprint(mqlkey, "/", topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i, ".", mqlvars.loop_i)
  823. mt.scopevars.Lock()
  824. if mt.scopevars.mql[mqlsn] == nil {
  825. mt.scopevars.mql[mqlsn] = &Variables{
  826. vars: map[string]interface{}{},
  827. loop_count: 1,
  828. loop_from: 1,
  829. loop_step: 1}
  830. }
  831. mt.scopevars.Unlock()
  832. //
  833. mt.scopevars.Lock()
  834. top_loopi := mt.scopevars.top.loop_from + (topvars.loop_i-1)*mt.scopevars.top.loop_step
  835. mt.scopevars.top.vars["topi"] = top_loopi
  836. dir_loopi := mt.scopevars.dir[basedir].loop_from + (dirvars.loop_i-1)*mt.scopevars.dir[basedir].loop_step
  837. mt.scopevars.dir[basedir].vars["diri"] = dir_loopi
  838. mt.scopevars.dir[basedir].vars["topi"] = top_loopi
  839. file_loopi := mt.scopevars.file[ffpath].loop_from + (filevars.loop_i-1)*mt.scopevars.file[ffpath].loop_step
  840. mt.scopevars.file[ffpath].vars["filei"] = file_loopi
  841. mt.scopevars.file[ffpath].vars["diri"] = dir_loopi
  842. mt.scopevars.file[ffpath].vars["topi"] = top_loopi
  843. mql_loopi := mt.scopevars.mql[mqlkey].loop_from + (mqlvars.loop_i-1)*mt.scopevars.mql[mqlkey].loop_step
  844. mt.scopevars.mql[mqlkey].vars["mqli"] = mql_loopi
  845. mt.scopevars.mql[mqlkey].vars["filei"] = file_loopi
  846. mt.scopevars.mql[mqlkey].vars["diri"] = dir_loopi
  847. mt.scopevars.mql[mqlkey].vars["topi"] = top_loopi
  848. //
  849. mt.scopevars.mql[mqlsn].vars["mqli"] = mql_loopi
  850. mt.scopevars.mql[mqlsn].vars["filei"] = file_loopi
  851. mt.scopevars.mql[mqlsn].vars["diri"] = dir_loopi
  852. mt.scopevars.mql[mqlsn].vars["topi"] = top_loopi
  853. mt.scopevars.Unlock()
  854. mt.BeforeRunAction(basedir, ffpath, mqlkey, mqlsn, staticactions)
  855. formatedmqlstr := mqr.FormatedQueryString
  856. mql := mt.ReplaceLoopSN(formatedmqlstr,
  857. global, topvars, dirvars, filevars, mqlvars,
  858. top_loopi, dir_loopi, file_loopi, mql_loopi,
  859. basedir, ffpath, mqlkey, mqlsn,
  860. )
  861. mqlreplace := rereplace_nbytes.FindAllStringSubmatch(mql, -1)
  862. for _, mqr := range mqlreplace {
  863. if len(mqr) == 2 {
  864. bs := make([]byte, cast.ToInt(mqr[1]))
  865. for i := 0; i < len(bs); i++ {
  866. bs[i] = byte(32 + rand.Intn(91))
  867. }
  868. s := string(bs)
  869. s = strings.ReplaceAll(s, ")", string([]byte{123}))
  870. s = strings.ReplaceAll(s, "'", string([]byte{124}))
  871. s = strings.ReplaceAll(s, "\"", string([]byte{125}))
  872. s = strings.ReplaceAll(s, "\\", string([]byte{126}))
  873. mql = strings.Replace(mql, mqr[0], s, 1)
  874. }
  875. }
  876. for i, sat := range staticactionexprs {
  877. mql = strings.Replace(mql, "["+strconv.Itoa(i)+"]", sat.SourceText, 1)
  878. }
  879. mql = strings.ReplaceAll(mql, "[[]", "[")
  880. mqri := &MQLRequestInstance{
  881. MQLRequest: mqr,
  882. PreparedQueryString: mql,
  883. }
  884. //
  885. ok_chan := make(chan error, 1)
  886. parallel_chan := make(chan bool, 1)
  887. parallel := false
  888. parallelcount := 0
  889. done := false
  890. wg.Add(1)
  891. go func(mqlvars *CurrentVars) {
  892. defer func() {
  893. atomic.AddInt32(&mqlcount, mqlvars.mqlcount)
  894. wg.Done()
  895. }()
  896. ch_ok := make(chan error)
  897. go func() {
  898. for {
  899. select {
  900. case parallelcount = <-ch_parallel_count:
  901. if !done && !parallel {
  902. parallel = true
  903. // 加入并发控制队列
  904. if parallelcount > 0 {
  905. if parallelcount > parallel_queue.Size() {
  906. parallel_queue.Growth(parallelcount)
  907. }
  908. parallel_queue.Push(1)
  909. }
  910. parallel_chan <- true
  911. }
  912. case ok := <-ch_ok:
  913. ok_chan <- ok
  914. if parallel {
  915. if parallelcount > 0 {
  916. // 从并发控制队列中移除
  917. parallel_queue.Pop()
  918. }
  919. } else {
  920. done = true
  921. }
  922. return
  923. }
  924. }
  925. }()
  926. nstaticactionexprs := map[int]*Action{}
  927. for i, sat := range staticactionexprs {
  928. nstaticactionexprs[i] = sat
  929. }
  930. // logger.Info(mqlkey, "第", fmt.Sprint(all_loop_i, ".", dir_loop_i, ".", file_loop_i, ".", curval.mql_i), "次执行开始")
  931. ch_ok <- mt.RunMQL(t, ctx,
  932. global,
  933. topvars,
  934. dirvars,
  935. filevars,
  936. mqlvars,
  937. basedir, ffpath, mqlkey, mqlsn, mqri, staticactions, nstaticactionexprs)
  938. // logger.Info(mqlkey, "第", fmt.Sprint(all_loop_i, ".", dir_loop_i, ".", file_loop_i, ".", curval.mql_i), "次执行结束")
  939. }(mqlvars)
  940. var err error
  941. select {
  942. case err = <-ok_chan: // 非并发,等待完成
  943. // logger.Info(mqlkey, "第", fmt.Sprint(all_loop_i, ".", dir_loop_i, ".", file_loop_i, ".", curval.mql_i), "次顺序执行完成")
  944. case <-parallel_chan: // 并发,执行继续下一次
  945. // logger.Info(mqlkey, "第", fmt.Sprint(all_loop_i, ".", dir_loop_i, ".", file_loop_i, ".", curval.mql_i), "次并发执行继续")
  946. }
  947. if err != nil {
  948. // logger.Error(err)
  949. break
  950. }
  951. }
  952. wg.Wait()
  953. mt.scopevars.RLock()
  954. loop_count := mt.scopevars.mql[mqlkey].loop_count
  955. mt.scopevars.RUnlock()
  956. if loop_count > 1 {
  957. ut := time.Since(st)
  958. sn := fmt.Sprint(topvars.loop_i, ".", dirvars.loop_i, ".", filevars.loop_i)
  959. as := ""
  960. if topvars.sleeptime > 0 {
  961. as = fmt.Sprint(", sleep ", topvars.sleeptime)
  962. }
  963. logger.Info(fmt.Sprint("mql ", mqlkey+"/"+sn, " loop ", loop_i, " times, usetime ", ut, as))
  964. }
  965. })
  966. }
  967. func (mt *MQLTest) getValue(basedir, ffpath, mqlkey, mqlsn string,
  968. staticactions *StaticActions, a any) (ak string, va any) {
  969. switch av := a.(type) {
  970. case *Action:
  971. switch av.Name {
  972. case "<EQ>":
  973. if len(av.Args) >= 2 {
  974. _, x := mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, av.Args[0])
  975. _, y := mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, av.Args[1])
  976. if cast.ToString(x) == cast.ToString(y) {
  977. return "", true
  978. }
  979. }
  980. return "", false
  981. }
  982. case *qstru.Expr:
  983. ak = av.VarName
  984. if ak == "now" {
  985. return "", time.Now()
  986. }
  987. mt.scopevars.Lock()
  988. switch staticactions.Scope {
  989. case "top":
  990. va = mt.scopevars.top.vars[ak]
  991. case "dir":
  992. va = mt.scopevars.dir[basedir].vars[ak]
  993. case "file":
  994. va = mt.scopevars.file[ffpath].vars[ak]
  995. default:
  996. var has bool
  997. va, has = mt.scopevars.mql[mqlsn].vars[ak]
  998. if !has {
  999. va = mt.scopevars.mql[mqlkey].vars[ak]
  1000. }
  1001. }
  1002. mt.scopevars.Unlock()
  1003. case string:
  1004. va = av
  1005. for _, tf := range time_layouts {
  1006. t, e := time.Parse(tf, av)
  1007. if e == nil {
  1008. va = t
  1009. break
  1010. }
  1011. }
  1012. default:
  1013. va = a
  1014. }
  1015. return
  1016. }
  1017. func (mt *MQLTest) setValue(basedir, ffpath, mqlkey, mqlsn string,
  1018. staticactions *StaticActions, ak string, va any) {
  1019. mt.scopevars.Lock()
  1020. switch staticactions.Scope {
  1021. case "top":
  1022. mt.scopevars.top.vars[ak] = va
  1023. case "dir":
  1024. mt.scopevars.dir[basedir].vars[ak] = va
  1025. case "file":
  1026. mt.scopevars.file[ffpath].vars[ak] = va
  1027. default:
  1028. _, has := mt.scopevars.mql[mqlsn].vars[ak]
  1029. if has {
  1030. mt.scopevars.mql[mqlsn].vars[ak] = va
  1031. } else {
  1032. mt.scopevars.mql[mqlkey].vars[ak] = va
  1033. }
  1034. }
  1035. mt.scopevars.Unlock()
  1036. }
  1037. func (mt *MQLTest) BeforeRunAction(basedir, ffpath, mqlkey, mqlsn string,
  1038. staticactions *StaticActions,
  1039. ) {
  1040. mt.runAction(basedir, ffpath, mqlkey, mqlsn, staticactions, staticactions.BeforeRunActions...)
  1041. }
  1042. func (mt *MQLTest) runAction(basedir, ffpath, mqlkey, mqlsn string,
  1043. staticactions *StaticActions, actions ...*Action,
  1044. ) {
  1045. for _, act := range actions {
  1046. switch act.Name {
  1047. case "set":
  1048. var a, b any
  1049. var ak string
  1050. if len(act.Args) > 1 {
  1051. a = act.Args[0]
  1052. b = act.Args[1]
  1053. }
  1054. ak, _ = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, a)
  1055. _, vb := mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, b)
  1056. mt.setValue(basedir, ffpath, mqlkey, mqlsn, staticactions, ak, vb)
  1057. case "add":
  1058. var a, b, va, vb any
  1059. var ak string
  1060. if len(act.Args) > 1 {
  1061. a = act.Args[0]
  1062. b = act.Args[1]
  1063. }
  1064. ak, va = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, a)
  1065. _, vb = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, b)
  1066. switch v := va.(type) {
  1067. case time.Time:
  1068. unit := "second"
  1069. if len(act.Args) > 2 {
  1070. unit = cast.ToString(act.Args[2])
  1071. }
  1072. vb := cast.ToInt(vb)
  1073. d := time.Duration(0)
  1074. switch unit {
  1075. case "second", "seconds", "s", "秒":
  1076. d = time.Duration(vb) * time.Second
  1077. case "minute", "minutes", "m", "分":
  1078. d = time.Duration(vb) * time.Minute
  1079. case "hour", "hours", "h", "时":
  1080. d = time.Duration(vb) * time.Hour
  1081. }
  1082. v = v.Add(d)
  1083. mt.setValue(basedir, ffpath, mqlkey, mqlsn, staticactions, ak, v)
  1084. }
  1085. case "mod":
  1086. var a, b, va, vb any
  1087. var ak string
  1088. if len(act.Args) > 1 {
  1089. a = act.Args[0]
  1090. b = act.Args[1]
  1091. }
  1092. ak, va = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, a)
  1093. _, vb = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, b)
  1094. av := cast.ToInt(va)
  1095. bv := cast.ToInt(vb)
  1096. v := av % bv
  1097. mt.setValue(basedir, ffpath, mqlkey, mqlsn, staticactions, ak, v)
  1098. case "case":
  1099. var a, b, va any
  1100. if len(act.Args) > 1 {
  1101. a = act.Args[0]
  1102. b = act.Args[1]
  1103. }
  1104. _, va = mt.getValue(basedir, ffpath, mqlkey, mqlsn, staticactions, a)
  1105. if cast.ToBool(va) {
  1106. switch vb := b.(type) {
  1107. case *Action:
  1108. mt.runAction(basedir, ffpath, mqlkey, mqlsn, staticactions, vb)
  1109. }
  1110. }
  1111. }
  1112. }
  1113. }
  1114. func (mt *MQLTest) InitScopeVars(basedir, ffpath, mqlkey string,
  1115. staticactions *StaticActions,
  1116. ) {
  1117. mt.scopevars.Lock()
  1118. if mt.scopevars.mql[mqlkey] == nil {
  1119. mt.scopevars.mql[mqlkey] = &Variables{
  1120. vars: map[string]interface{}{},
  1121. loop_count: 1,
  1122. loop_from: 1,
  1123. loop_step: 1}
  1124. }
  1125. mt.scopevars.Unlock()
  1126. for k, v := range staticactions.Variables {
  1127. mt.scopevars.Lock()
  1128. switch staticactions.Scope {
  1129. case "top":
  1130. mt.scopevars.top.vars[k] = v
  1131. case "dir":
  1132. mt.scopevars.dir[basedir].vars[k] = v
  1133. case "file":
  1134. mt.scopevars.file[ffpath].vars[k] = v
  1135. default:
  1136. mt.scopevars.mql[mqlkey].vars[k] = v
  1137. }
  1138. mt.scopevars.Unlock()
  1139. }
  1140. if staticactions.Timeout != nil {
  1141. mt.scopevars.Lock()
  1142. switch staticactions.Scope {
  1143. case "top":
  1144. mt.scopevars.top.timeout = *staticactions.Timeout
  1145. case "dir":
  1146. mt.scopevars.dir[basedir].timeout = *staticactions.Timeout
  1147. case "file":
  1148. mt.scopevars.file[ffpath].timeout = *staticactions.Timeout
  1149. default:
  1150. mt.scopevars.mql[mqlkey].timeout = *staticactions.Timeout
  1151. }
  1152. mt.scopevars.Unlock()
  1153. }
  1154. if staticactions.Qmeta != nil {
  1155. mt.scopevars.Lock()
  1156. switch staticactions.Scope {
  1157. case "top":
  1158. mt.scopevars.top.qmeta = staticactions.Qmeta
  1159. case "dir":
  1160. mt.scopevars.dir[basedir].qmeta = staticactions.Qmeta
  1161. case "file":
  1162. mt.scopevars.file[ffpath].qmeta = staticactions.Qmeta
  1163. default:
  1164. mt.scopevars.mql[mqlkey].qmeta = staticactions.Qmeta
  1165. }
  1166. mt.scopevars.Unlock()
  1167. }
  1168. if staticactions.LoopCount != nil {
  1169. mt.scopevars.Lock()
  1170. switch staticactions.Scope {
  1171. case "top":
  1172. mt.scopevars.top.loop_count = *staticactions.LoopCount
  1173. case "dir":
  1174. mt.scopevars.dir[basedir].loop_count = *staticactions.LoopCount
  1175. case "file":
  1176. mt.scopevars.file[ffpath].loop_count = *staticactions.LoopCount
  1177. default:
  1178. mt.scopevars.mql[mqlkey].loop_count = *staticactions.LoopCount
  1179. }
  1180. mt.scopevars.Unlock()
  1181. }
  1182. if staticactions.LoopFrom != nil {
  1183. mt.scopevars.Lock()
  1184. switch staticactions.Scope {
  1185. case "top":
  1186. mt.scopevars.top.loop_from = *staticactions.LoopFrom
  1187. case "dir":
  1188. mt.scopevars.dir[basedir].loop_from = *staticactions.LoopFrom
  1189. case "file":
  1190. mt.scopevars.file[ffpath].loop_from = *staticactions.LoopFrom
  1191. default:
  1192. mt.scopevars.mql[mqlkey].loop_from = *staticactions.LoopFrom
  1193. }
  1194. mt.scopevars.Unlock()
  1195. }
  1196. if staticactions.LoopStep != nil {
  1197. mt.scopevars.Lock()
  1198. switch staticactions.Scope {
  1199. case "top":
  1200. mt.scopevars.top.loop_step = *staticactions.LoopStep
  1201. case "dir":
  1202. mt.scopevars.dir[basedir].loop_step = *staticactions.LoopStep
  1203. case "file":
  1204. mt.scopevars.file[ffpath].loop_step = *staticactions.LoopStep
  1205. default:
  1206. mt.scopevars.mql[mqlkey].loop_step = *staticactions.LoopStep
  1207. }
  1208. mt.scopevars.Unlock()
  1209. }
  1210. }
  1211. func (mt *MQLTest) RunMQL(t *testing.T, ctx context.Context,
  1212. global *GlobalVars,
  1213. topvars *CurrentVars,
  1214. dirvars *CurrentVars,
  1215. filevars *CurrentVars,
  1216. mqlvars *CurrentVars,
  1217. basedir, ffpath, mqlkey, mqlsn string, mqri *MQLRequestInstance, staticactions *StaticActions, staticactionexprs map[int]*Action) error {
  1218. mqlstr := mqri.PreparedQueryString
  1219. if staticactions.WaitName != nil {
  1220. global.Lock()
  1221. wgs := []*sync.WaitGroup{}
  1222. if *staticactions.WaitName == "" {
  1223. for _, wg := range global.wg_wait_fork_routine {
  1224. wgs = append(wgs, wg)
  1225. }
  1226. } else {
  1227. wg := global.wg_wait_fork_routine[*staticactions.WaitName]
  1228. if wg != nil {
  1229. wgs = append(wgs, wg)
  1230. }
  1231. }
  1232. global.Unlock()
  1233. for _, wg := range wgs {
  1234. wg.Wait()
  1235. }
  1236. }
  1237. if staticactions.ParallelCount != nil {
  1238. mt.scopevars.Lock()
  1239. switch staticactions.Scope {
  1240. case "top":
  1241. topvars.ch_parallel_count <- *staticactions.ParallelCount
  1242. case "dir":
  1243. dirvars.ch_parallel_count <- *staticactions.ParallelCount
  1244. case "file":
  1245. filevars.ch_parallel_count <- *staticactions.ParallelCount
  1246. default:
  1247. mqlvars.ch_parallel_count <- *staticactions.ParallelCount
  1248. }
  1249. mt.scopevars.Unlock()
  1250. }
  1251. // 重新获取修正后的动作
  1252. actionexprs, e := getActionExprs(mqlstr, staticactionexprs)
  1253. if !assert.Nil(t, e, e) {
  1254. return e
  1255. }
  1256. dynamicactions := actionexprs.DynamicActions()
  1257. if len(dynamicactions.SubscribeArgs) > 0 {
  1258. subscribe(dynamicactions.SubscribeArgs...)
  1259. }
  1260. if len(dynamicactions.UnsubscribeArgs) > 0 {
  1261. unsubscribe(dynamicactions.UnsubscribeArgs...)
  1262. }
  1263. values := []interface{}{}
  1264. if len(dynamicactions.Params) > 0 {
  1265. err := json.Unmarshal([]byte(dynamicactions.Params), &values)
  1266. if err != nil {
  1267. assert.Nil(t, fmt.Sprint("params参数只支持JSON Array,", dynamicactions.Params, mqlstr), err)
  1268. return err
  1269. }
  1270. }
  1271. x := atomic.AddInt32(&global.mqlcount, 1)
  1272. atomic.AddInt32(&topvars.mqlcount, 1)
  1273. atomic.AddInt32(&dirvars.mqlcount, 1)
  1274. atomic.AddInt32(&filevars.mqlcount, 1)
  1275. atomic.AddInt32(&mqlvars.mqlcount, 1)
  1276. err := mt.RunMQLTryDo(t, ctx,
  1277. global,
  1278. topvars,
  1279. dirvars,
  1280. filevars,
  1281. mqlvars,
  1282. basedir, ffpath, mqlkey, mqlsn+"("+strconv.Itoa(int(x))+")", mqri, values, staticactions, actionexprs)
  1283. if err != nil {
  1284. // 执行过程有错,停止继续执行
  1285. return err
  1286. }
  1287. if staticactions.SleepTime != nil {
  1288. global.sleeptime += *staticactions.SleepTime
  1289. topvars.sleeptime += *staticactions.SleepTime
  1290. dirvars.sleeptime += *staticactions.SleepTime
  1291. filevars.sleeptime += *staticactions.SleepTime
  1292. mqlvars.sleeptime += *staticactions.SleepTime
  1293. time.Sleep(*staticactions.SleepTime)
  1294. }
  1295. return nil
  1296. }