Skip to content

Commit 313c07a

Browse files
zorinozorino
authored
Merge pull request #4 from zorino/update-to-badger-v3
Update to badger v3
2 parents f914134 + 2b72ce1 commit 313c07a

File tree

18 files changed

+152
-127
lines changed

18 files changed

+152
-127
lines changed

‎api/server.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ import (
3030
"strings"
3131
"time"
3232

33-
"github.com/dgraph-io/badger/options"
3433
"github.com/go-chi/chi"
3534
"github.com/golang/protobuf/proto"
3635
"github.com/rs/xid"
@@ -45,7 +44,7 @@ var dbStats *kvstore.KStats
4544
var tmpFolder = "/tmp/"
4645
var nbOfThreads = 0
4746

48-
func NewServer(dbPath string, portNumber int, tableLoadingMode options.FileLoadingMode, valueLoadingMode options.FileLoadingMode, newNbThreads int, newTmpFolder string) {
47+
func NewServer(dbPath string, portNumber int, newNbThreads int, newTmpFolder string) {
4948

5049
runtime.GOMAXPROCS(512)
5150

@@ -63,7 +62,7 @@ func NewServer(dbPath string, portNumber int, tableLoadingMode options.FileLoadi
6362
fmt.Printf(" + Opening kAAmer Database.. ")
6463
startTime := time.Now()
6564

66-
kvStores = kvstore.KVStoresNew(dbPath, 12, tableLoadingMode, valueLoadingMode, true, false, true)
65+
kvStores = kvstore.KVStoresNew(dbPath, 12, true, false, true)
6766
defer kvStores.Close()
6867

6968
dbStatsByte, ok := kvStores.ProteinStore.GetValue([]byte("db_stats"))

‎cmd/kaamer-bench/main.go

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"runtime"
2424
"sync"
2525

26-
"github.com/dgraph-io/badger/options"
2726
"github.com/pkg/profile"
2827
"github.com/zorino/kaamer/pkg/kvstore"
2928
"github.com/zorino/kaamer/pkg/makedb"
@@ -56,7 +55,7 @@ var (
5655
makedbLenght = flag.Uint("length", uint(MaxInt), "process x number of files")
5756
noIndex = flag.Bool("noindex", false, "prevent the indexing of database")
5857

59-
LoadingMode = map[string]options.FileLoadingMode{"memorymap": options.MemoryMap, "fileio": options.FileIO}
58+
// LoadingMode = map[string]options.FileLoadingMode{"memorymap": options.MemoryMap, "fileio": options.FileIO}
6059
)
6160

6261
func main() {
@@ -74,8 +73,7 @@ func main() {
7473
-p port (default: 8321)
7574
-t number of threads to use (default all)
7675
-tmp tmp folder for query import (default /tmp)
77-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
78-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
76+
7977
(flag)
8078
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
8179
(to limit the number of open files)
@@ -88,8 +86,7 @@ func main() {
8886
-t number of threads to use (default all)
8987
-offset start processing raw uniprot file at protein number x
9088
-length process x number of proteins (-1 == infinity)
91-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
92-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
89+
9390
(flag)
9491
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
9592
(to limit the number of open files)
@@ -116,18 +113,16 @@ func main() {
116113
}
117114

118115
/* Setting values from CLI */
119-
var tableLoadingMode options.FileLoadingMode
120-
var valueLoadingMode options.FileLoadingMode
121-
var ok = false
116+
// var ok = false
122117

123-
if tableLoadingMode, ok = LoadingMode[*tableMode]; !ok {
124-
fmt.Println("TableMode unrecognized ! use fileio or memorymap!")
125-
os.Exit(1)
126-
}
127-
if valueLoadingMode, ok = LoadingMode[*valueMode]; !ok {
128-
fmt.Println("ValueMode unrecognized ! use fileio or memorymap!")
129-
os.Exit(1)
130-
}
118+
// if tableLoadingMode, ok = LoadingMode[*tableMode]; !ok {
119+
// fmt.Println("TableMode unrecognized ! use fileio or memorymap!")
120+
// os.Exit(1)
121+
// }
122+
// if valueLoadingMode, ok = LoadingMode[*valueMode]; !ok {
123+
// fmt.Println("ValueMode unrecognized ! use fileio or memorymap!")
124+
// os.Exit(1)
125+
// }
131126

132127
/* Functions */
133128
switch *function {
@@ -141,7 +136,7 @@ func main() {
141136
wg.Add(1)
142137
go NewMonitor(1, &stop, &wg)
143138
var kvStores *kvstore.KVStores
144-
kvStores = kvstore.KVStoresNew(*dbPath, *nbThreads, tableLoadingMode, valueLoadingMode, true, false, true)
139+
kvStores = kvstore.KVStoresNew(*dbPath, *nbThreads, true, false, true)
145140
stop = true
146141
wg.Wait()
147142
defer kvStores.Close()
@@ -161,7 +156,7 @@ func main() {
161156
var wg sync.WaitGroup
162157
wg.Add(1)
163158
go NewMonitor(10, &stop, &wg)
164-
makedb.NewMakedb(*dbPath, *inputPath, *inputFmt, *nbThreads, *makedbOffset, *makedbLenght, *maxSize, tableLoadingMode, valueLoadingMode, *noIndex)
159+
makedb.NewMakedb(*dbPath, *inputPath, *inputFmt, *nbThreads, *makedbOffset, *makedbLenght, *maxSize, *noIndex)
165160
stop = true
166161
wg.Wait()
167162
}

‎cmd/kaamer-db/main.go

Lines changed: 12 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ import (
2222
"os"
2323
"runtime"
2424

25-
"github.com/dgraph-io/badger/options"
26-
"github.com/zorino/kaamer/api"
25+
server "github.com/zorino/kaamer/api"
2726
"github.com/zorino/kaamer/pkg/backupdb"
2827
"github.com/zorino/kaamer/pkg/downloaddb"
2928
"github.com/zorino/kaamer/pkg/gcdb"
@@ -37,10 +36,6 @@ const (
3736
MaxInt uint32 = 1<<32 - 1
3837
)
3938

40-
var (
41-
LoadingMode = map[string]options.FileLoadingMode{"memorymap": options.MemoryMap, "fileio": options.FileIO}
42-
)
43-
4439
func main() {
4540

4641
usage := `
@@ -55,9 +50,6 @@ func main() {
5550
-t number of threads to use (default all)
5651
-tmp tmp folder for query import (default /tmp)
5752
58-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
59-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
60-
6153
// Database
6254
6355
-make make the protein database
@@ -68,8 +60,7 @@ func main() {
6860
-t number of threads to use (default all)
6961
-offset start processing raw uniprot file at protein number x
7062
-length process x number of proteins (-1 == infinity)
71-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
72-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
63+
7364
(flag)
7465
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
7566
(to limit the number of open files)
@@ -79,8 +70,7 @@ func main() {
7970
(input)
8071
-d database directory
8172
-t number of threads to use (default all)
82-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
83-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
73+
8474
(flag)
8575
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
8676
(to limit the number of open files)
@@ -108,8 +98,7 @@ func main() {
10898
(input)
10999
-dbs databases directory
110100
-o output directory of merged database
111-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
112-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
101+
113102
(flag)
114103
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
115104
(to limit the number of open files)
@@ -118,15 +107,12 @@ func main() {
118107
(input)
119108
-d badger db directory
120109
-o badger backup output directory
121-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
122-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
123110
124111
-restore restore a backup database
125112
(input)
126113
-d badger backup db directory
127114
-o badger db output directory
128-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
129-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
115+
130116
(flag)
131117
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
132118
(to limit the number of open files)
@@ -136,8 +122,7 @@ func main() {
136122
-d database directory
137123
-it number of GC iterations
138124
-ratio number of ratio of the GC (between 0-1)
139-
-tableMode (fileio, memorymap) default memorymap / fileio decreases memory usage
140-
-valueMode (fileio, memorymap) default memorymap / fileio decreases memory usage
125+
141126
(flag)
142127
-maxsize will maximize the size of tables (.sst) and vlog (.log) files
143128
(to limit the number of open files)
@@ -156,8 +141,6 @@ func main() {
156141
var makedbOffset = flag.Uint("offset", 0, "offset to process raw file")
157142
var makedbLenght = flag.Uint("length", uint(MaxInt), "process x number of files")
158143
var maxSize = flag.Bool("maxsize", false, "to maximize badger output file size")
159-
var tableMode = flag.String("tablemode", "memorymap", "table loading mode (fileio, memorymap)")
160-
var valueMode = flag.String("valuemode", "memorymap", "value loading mode (fileio, memorymap)")
161144
var noIndex = flag.Bool("noindex", false, "prevent the indexing of database")
162145

163146
var indexOpt = flag.Bool("index", false, "program")
@@ -187,20 +170,6 @@ func main() {
187170
}
188171
flag.Parse()
189172

190-
/* Setting values from CLI */
191-
var tableLoadingMode options.FileLoadingMode
192-
var valueLoadingMode options.FileLoadingMode
193-
var ok = false
194-
195-
if tableLoadingMode, ok = LoadingMode[*tableMode]; !ok {
196-
fmt.Println("TableMode unrecognized ! use fileio or memorymap!")
197-
os.Exit(1)
198-
}
199-
if valueLoadingMode, ok = LoadingMode[*valueMode]; !ok {
200-
fmt.Println("ValueMode unrecognized ! use fileio or memorymap!")
201-
os.Exit(1)
202-
}
203-
204173
if _, err := os.Stat(*tmpFolder); os.IsNotExist(err) {
205174
fmt.Printf("Directory %s does not exist !\n", tmpFolder)
206175
os.Exit(1)
@@ -211,7 +180,7 @@ func main() {
211180
if *dbPath == "" {
212181
fmt.Println("No db path !")
213182
} else {
214-
server.NewServer(*dbPath, *portNumber, tableLoadingMode, valueLoadingMode, *nbThreads, *tmpFolder)
183+
server.NewServer(*dbPath, *portNumber, *nbThreads, *tmpFolder)
215184
}
216185
os.Exit(0)
217186
}
@@ -268,7 +237,7 @@ func main() {
268237
fmt.Println("No input format (-f) !")
269238
os.Exit(1)
270239
} else {
271-
makedb.NewMakedb(*dbPath, *inputPath, *inputFmt, *nbThreads, *makedbOffset, *makedbLenght, *maxSize, tableLoadingMode, valueLoadingMode, *noIndex)
240+
makedb.NewMakedb(*dbPath, *inputPath, *inputFmt, *nbThreads, *makedbOffset, *makedbLenght, *maxSize, *noIndex)
272241
}
273242

274243
os.Exit(0)
@@ -280,7 +249,7 @@ func main() {
280249
fmt.Println("No db path !")
281250
os.Exit(1)
282251
} else {
283-
indexdb.NewIndexDB(*dbPath, *nbThreads, *maxSize, tableLoadingMode, valueLoadingMode)
252+
indexdb.NewIndexDB(*dbPath, *nbThreads, *maxSize)
284253
}
285254

286255
os.Exit(0)
@@ -290,7 +259,7 @@ func main() {
290259
if *dbsPath == "" || *outPath == "" {
291260
fmt.Println("Need to have a valid databases path !")
292261
} else {
293-
mergedb.NewMergedb(*dbsPath, *outPath, *maxSize, tableLoadingMode, valueLoadingMode)
262+
mergedb.NewMergedb(*dbsPath, *outPath, *maxSize)
294263
}
295264
os.Exit(0)
296265
}
@@ -299,7 +268,7 @@ func main() {
299268
if *dbPath == "" {
300269
fmt.Println("No db path !")
301270
} else {
302-
gcdb.NewGC(*dbPath, *gcIteration, *gcRatio, *maxSize, tableLoadingMode, valueLoadingMode)
271+
gcdb.NewGC(*dbPath, *gcIteration, *gcRatio, *maxSize)
303272
}
304273
os.Exit(0)
305274
}
@@ -310,7 +279,7 @@ func main() {
310279
} else if *outPath == "" {
311280
fmt.Println("Need to have a valid backup directory path !")
312281
} else {
313-
backupdb.Backupdb(*dbPath, *outPath, tableLoadingMode, valueLoadingMode)
282+
backupdb.Backupdb(*dbPath, *outPath)
314283
}
315284
os.Exit(0)
316285
}

‎go.mod

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
77
github.com/OneOfOne/xxhash v1.2.5
88
github.com/biogo/biogo v1.0.1
9-
github.com/dgraph-io/badger v1.6.0
9+
github.com/dgraph-io/badger/v3 v3.2103.0
1010
github.com/dustin/go-humanize v1.0.0
1111
github.com/go-chi/chi v4.0.2+incompatible
1212
github.com/golang/protobuf v1.3.2
@@ -17,8 +17,6 @@ require (
1717
github.com/spf13/pflag v1.0.5 // indirect
1818
github.com/stretchr/testify v1.4.0 // indirect
1919
github.com/zorino/counters v0.0.0-20190409141417-6ea0e007435b
20-
golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3
21-
golang.org/x/sys v0.0.0-20190927073244-c990c680b611 // indirect
22-
golang.org/x/text v0.3.2 // indirect
23-
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
20+
golang.org/x/net v0.0.0-20201021035429-f5854403a974
21+
github.com/lanrat/extsort v1.0.0
2422
)

0 commit comments

Comments
 (0)