odbtest/test/testdfs.go

package main

import (
	"io/ioutil"
	"log"
	"os"
	"runtime/pprof"

	"gopkg.in/src-d/go-billy.v4"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	g, err := odb.New(&odb.Option{Cache: odb.CacheAll})
	if err != nil {
		log.Fatal(err)
	}

	profFile, err := os.OpenFile("testrun_cpu.prof", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
	if err != nil {
		log.Fatal(err)
	}
	defer profFile.Close()
	if err = pprof.StartCPUProfile(profFile); err != nil {
		log.Fatal(err)
	}
	defer pprof.StopCPUProfile()

	files, err := getExportFiles("/app/matrix/m3mcc", true, g, odb.NewDFS(g))
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Files", len(files))
}

func getExportFiles(fileName string, isDir bool, g *odb.Gutil, fs billy.Filesystem) (map[string][]byte, error) {
	files := make(map[string][]byte)

	if isDir {
		fis, err := fs.ReadDir(fileName)
		if err != nil {
			return nil, err
		}
		files[fileName+"/"] = []byte("") // empty dir
		for _, f := range fis {
			fname := fs.Join(fileName, f.Name())
			subFiles, err := getExportFiles(fname, f.IsDir(), g, fs)
			if err != nil {
				return nil, err
			}
			for k, v := range subFiles {
				files[k] = v
			}
		}
	} else {
		file, err := fs.Open(fileName)
		if err != nil {
			return nil, err
		}
		b, err := ioutil.ReadAll(file)
		if err != nil {
			return nil, err
		}
		files[fileName] = b
	}

	return files, nil
}