cgimport/reader/txtreader.go

package reader

import (
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"strings"

	"git.wecise.com/wecise/cgimport/schema"
)

type TXTBlockReader struct {
	*LineReader
	filetype  schema.FileType
	firstline string
	nextline  string
}

func NewTXTBlockReader(filename string, filetype schema.FileType, reader io.Reader) *TXTBlockReader {
	return &TXTBlockReader{
		LineReader: NewLineReader(filename, reader),
		filetype:   filetype,
	}
}

var regrecordstart = regexp.MustCompile(`^(?:[\.\/a-zA-Z0-9_]*:)?V:(\{.*)`)
var regrecordend = regexp.MustCompile(`\}\s*$`)

func (br *TXTBlockReader) ReadBlock(skiplines int) (block map[string]any, line string, linecount int, err error) {
	eof := false
	line = br.nextline
	for {
		for {
			br.nextline, linecount, eof, err = br.ReadLine()
			if err != nil {
				return
			}
			if br.nextline == "" && eof {
				if line == "" {
					return
				}
				break
			}
			if linecount <= skiplines {
				continue
			}
			if regrecordend.MatchString(line) {
				break
			}
			if regrecordstart.MatchString(br.nextline) {
				break
			}
			line += br.nextline
		}
		linecount--
		if !regrecordstart.MatchString(line) || !regrecordend.MatchString(line) {
			if line != "" {
				logger.Info(fmt.Sprint("skip non-json line ", br.filename, ":", linecount, " ", line))
			}
			if linecount == 1 {
				br.firstline = line
			}
			line = br.nextline
			continue
		}
		line = regrecordstart.ReplaceAllString(line, "$1")
		line = strings.ReplaceAll(line, `\`, `\\`)
		line = strings.ReplaceAll(line, "\t", `\t`)
		line = strings.ReplaceAll(line, "\r", `\r`)
		line = strings.ReplaceAll(line, "\n", `\n`)
		block = map[string]any{}
		err = json.Unmarshal([]byte(line), &block)
		if err != nil {
			return
		}
		return
	}
}