Merge pull request #2 from filedrive-team/feature

Retrieve file from CAR

Author: qianhh

.gitignore

@@ -15,3 +15,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+.idea

Makefile

@@ -0,0 +1,3 @@
+.PHONY: build
+build:
+	go build -ldflags "-s -w" -o graphsplit graphsplit.go utils.go retrieve.go

README.md

@@ -11,7 +11,7 @@ Graphsplit has solved the problem we faced above. It takes advantage of IPLD con
 
 ## Build
 ```sh
-go build -o graphsplit graphsplit.go utils.go
+make
 ```
 
 ## Usage
@@ -43,6 +43,17 @@ Import car file to IPFS:
 ipfs dag import /path/to/car-dir/car-file
 ```
 
+Retrieve files:
+```sh
+# car-path: directory or file, in form of .car
+# output-dir: usually just be the same as /path/to/output-dir
+# parallel: number goroutines run when retrieving
+./graphsplit retrieve \
+--car-path=/path/to/car-path \
+--output-dir=/path/to/output-dir \
+--parallel=2
+```
+
 ## Contribute
 
 PRs are welcome!

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/ipfs/go-ipfs-blockstore v1.0.3
 	github.com/ipfs/go-ipfs-chunker v0.0.5
 	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
+	github.com/ipfs/go-ipfs-files v0.0.8
 	github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 // indirect
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/ipfs/go-log/v2 v2.1.2

graphsplit.go

@@ -3,12 +3,11 @@ package main
 import (
 	"context"
 	"fmt"
-	"os"
-
 	logging "github.com/ipfs/go-log/v2"
 	_ "github.com/jinzhu/gorm/dialects/mysql"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
+	"os"
 )
 
 var log = logging.Logger("graphsplit")
@@ -17,6 +16,7 @@ func main() {
 	logging.SetLogLevel("*", "INFO")
 	local := []*cli.Command{
 		chunkCmd,
+		retrieveCmd,
 	}
 
 	app := &cli.App{
@@ -33,31 +33,32 @@ func main() {
 
 var chunkCmd = &cli.Command{
 	Name:  "chunk",
-	Usage: "",
+	Usage: "Generate CAR files of the specified size",
 	Flags: []cli.Flag{
 		&cli.Int64Flag{
 			Name:  "slice-size",
 			Value: 17179869184, // 16G
-			Usage: fmt.Sprintf("specify chunk piece size"),
+			Usage: "specify chunk piece size",
 		},
 		&cli.IntFlag{
 			Name:  "parallel",
 			Value: 4,
-			Usage: fmt.Sprintf("specify how many number of goroutines runs when generate file node"),
+			Usage: "specify how many number of goroutines runs when generate file node",
 		},
 		&cli.StringFlag{
 			Name:     "graph-name",
 			Required: true,
-			Usage:    fmt.Sprintf("specify graph name"),
+			Usage:    "specify graph name",
 		},
 		&cli.StringFlag{
 			Name:  "parent-path",
 			Value: "",
-			Usage: fmt.Sprintf("specify graph parent path"),
+			Usage: "specify graph parent path",
 		},
 		&cli.StringFlag{
 			Name:     "car-dir",
 			Required: true,
+			Usage:    "specify output CAR directory",
 		},
 	},
 	Action: func(c *cli.Context) error {
@@ -73,6 +74,9 @@ var chunkCmd = &cli.Command{
 		if sliceSize == 0 {
 			return xerrors.Errorf("Unexpected! Slice size has been set as 0")
 		}
+		if parallel <= 0 {
+			return xerrors.Errorf("Unexpected! Parallel has to be greater than 0")
+		}
 
 		args := c.Args().Slice()
 		sliceTotal := GetGraphCount(args, sliceSize)
@@ -154,7 +158,6 @@ var chunkCmd = &cli.Command{
 				}
 
 			}
-
 		}
 		if cumuSize > 0 {
 			// todo build ipld from graphFiles
@@ -166,3 +169,39 @@ var chunkCmd = &cli.Command{
 		return nil
 	},
 }
+
+var retrieveCmd = &cli.Command{
+	Name:  "retrieve",
+	Usage: "Retrieve files from CAR files",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "car-path",
+			Required: true,
+			Usage:    "specify source car path, directory or file",
+		},
+		&cli.StringFlag{
+			Name:     "output-dir",
+			Required: true,
+			Usage:    "specify output directory",
+		},
+		&cli.IntFlag{
+			Name:  "parallel",
+			Value: 4,
+			Usage: "specify how many number of goroutines runs when generate file node",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		parallel := c.Int("parallel")
+		outputDir := c.String("output-dir")
+		carPath := c.String("car-path")
+		if parallel <= 0 {
+			return xerrors.Errorf("Unexpected! Parallel has to be greater than 0")
+		}
+
+		CarTo(carPath, outputDir, parallel)
+		Merge(outputDir, parallel)
+
+		fmt.Println("completed!")
+		return nil
+	},
+}