update vendor deps; add discrete options and callback functions; add -strict-alt-files flag to index tool

Author: thisisaaronland

cmd/wof-sqlite-index-features/main.go

@@ -39,9 +39,9 @@ func main() {
 	live_hard := flag.Bool("live-hard-die-fast", true, "Enable various performance-related pragmas at the expense of possible (unlikely) database corruption")
 	timings := flag.Bool("timings", false, "Display timings during and after indexing")
 	optimize := flag.Bool("optimize", true, "Attempt to optimize the database before closing connection")
-	// liberal := flag.Bool("liberal", false, "Do not trigger errors for records that can not be processed, for whatever reason")
 
 	alt_files := flag.Bool("index-alt-files", false, "Index alt geometries")
+	strict_alt_files := flag.Bool("strict-alt-files", true, "Be strict when indexing alt geometries")
 
 	var procs = flag.Int("processes", (runtime.NumCPU() * 2), "The number of concurrent processes to index data with")
 
@@ -200,7 +200,12 @@ func main() {
 		logger.Fatal("You forgot to specify which (any) tables to index")
 	}
 
-	idx, err := index.NewDefaultSQLiteFeaturesIndexer(db, to_index)
+	opts := index.DefaultSQLiteFeaturesIndexerCallbackOptions()
+	opts.StrictAltFiles = *strict_alt_files
+
+	cb := index.SQLiteFeaturesIndexerCallback(opts)
+
+	idx, err := index.NewSQLiteFeaturesIndexerWithCallback(db, to_index, cb)
 
 	if err != nil {
 		logger.Fatal("failed to create sqlite indexer because %s", err)

go.mod

@@ -4,12 +4,12 @@ go 1.12
 
 require (
 	github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.0
-	github.com/whosonfirst/go-whosonfirst-index v0.2.2
+	github.com/whosonfirst/go-whosonfirst-index v0.2.3
 	github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972
 	github.com/whosonfirst/go-whosonfirst-index-sqlite v0.0.2
 	github.com/whosonfirst/go-whosonfirst-log v0.1.0
 	github.com/whosonfirst/go-whosonfirst-sqlite v0.1.4
-	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.4
+	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5
 	github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.0
 	github.com/whosonfirst/warning v0.1.0
 )

go.sum

@@ -112,6 +112,8 @@ github.com/whosonfirst/go-whosonfirst-index v0.1.2 h1:pm/NY4O21sN7PPrOsZNfuv7kX2
 github.com/whosonfirst/go-whosonfirst-index v0.1.2/go.mod h1:SfFN3GjmpS5TQK4mhvEcH+OfTSTWGLY8B6y48SmjLmQ=
 github.com/whosonfirst/go-whosonfirst-index v0.2.2 h1:P+dgqipNHgIX8a/u03sT3bHc/rqxopKIUn21mu0c/Bw=
 github.com/whosonfirst/go-whosonfirst-index v0.2.2/go.mod h1:n3rY2Hza4+MFtKES9C1sS4AAIkA5va/nagzvt+4ZFBw=
+github.com/whosonfirst/go-whosonfirst-index v0.2.3 h1:a+vfmTPp2GkyZExQqXzLTK2VwkjdnOyyKZHRbpMhB00=
+github.com/whosonfirst/go-whosonfirst-index v0.2.3/go.mod h1:n3rY2Hza4+MFtKES9C1sS4AAIkA5va/nagzvt+4ZFBw=
 github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972 h1:xPrg89CfHOG3u7eaW6fa9ke4Px5kErS21D4mBf6usXE=
 github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972/go.mod h1:TNMdhUDIVyCrcr9WziZRt9UARQAvM/T1x4h69gVkxsA=
 github.com/whosonfirst/go-whosonfirst-index-sqlite v0.0.2 h1:rmp0Nm6adNVPEZ7A/LlvNjK8ANHzLm6z03aNR9yQ028=
@@ -143,6 +145,8 @@ github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.3 h1:XqYn6pnl9J63FZ99
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.3/go.mod h1:zbwNL08/j8uJ6fpYMYrFK8Pb3ZoA/ZcAaNYvvFq5o9U=
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.4 h1:asuQX5Nj/nKXmxuiVKcfJTdvldEqjnDpm1VwwFB5Mc4=
 github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.4/go.mod h1:zbwNL08/j8uJ6fpYMYrFK8Pb3ZoA/ZcAaNYvvFq5o9U=
+github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5 h1:sicGwGk1YK26M9P+s197ZkPiuqG8jv7J9UP0UEBV5+4=
+github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5/go.mod h1:jXNvI/rNqrL9nXkTefSBChmepQuOn9YMb/xK1LDO+jA=
 github.com/whosonfirst/go-whosonfirst-sqlite-index v0.0.1 h1:/GuAgxIHn5vZwAEDA4XUSWe7rYWZW30I4HLQRTpBeJY=
 github.com/whosonfirst/go-whosonfirst-sqlite-index v0.0.1/go.mod h1:5fY1ikCDhe1u0umawonTOICPzv/2CgfFT9hTAen5nsI=
 github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.0 h1:/tljUy2rchRC9XE4FDteNmYZYhYqxWX2uF4c61WlocU=

index.go

@@ -11,10 +11,23 @@ import (
 	"github.com/whosonfirst/warning"
 	"io"
 	"io/ioutil"
-	_ "log"
+	"log"
 )
 
-func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table) (*sql_index.SQLiteIndexer, error) {
+type SQLiteFeaturesIndexerCallbackOptions struct {
+	StrictAltFiles bool
+}
+
+func DefaultSQLiteFeaturesIndexerCallbackOptions() *SQLiteFeaturesIndexerCallbackOptions {
+
+	opts := &SQLiteFeaturesIndexerCallbackOptions{
+		StrictAltFiles: true,
+	}
+
+	return opts
+}
+
+func SQLiteFeaturesIndexerCallback(opts *SQLiteFeaturesIndexerCallbackOptions) sql_index.SQLiteIndexerFunc {
 
 	cb := func(ctx context.Context, fh io.Reader, args ...interface{}) (interface{}, error) {
 
@@ -30,6 +43,12 @@ func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table
 				return nil, err
 			}
 
+			// we read the whole thing in to memory here so don't
+			// have to worry about whether fh is a ReadSeeker if
+			// the first attempt to load a primary WOF file fails
+			// and we try again for an alt file
+			// (20191103/straup)
+
 			body, err := ioutil.ReadAll(fh)
 
 			if err != nil {
@@ -43,7 +62,14 @@ func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table
 				alt, alt_err := feature.NewWOFAltFeature(body)
 
 				if alt_err != nil && !warning.IsWarning(alt_err) {
+
 					msg := fmt.Sprintf("Unable to load %s, because %s (%s)", path, alt_err, err)
+
+					if !opts.StrictAltFiles {
+						log.Printf("%s - SKIPPING\n", msg)
+						return nil, nil
+					}
+
 					return nil, errors.New(msg)
 				}
 
@@ -54,5 +80,17 @@ func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table
 		}
 	}
 
+	return cb
+}
+
+func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table) (*sql_index.SQLiteIndexer, error) {
+
+	opts := DefaultSQLiteFeaturesIndexerCallbackOptions()
+	cb := SQLiteFeaturesIndexerCallback(opts)
+
+	return NewSQLiteFeaturesIndexerWithCallback(db, to_index, cb)
+}
+
+func NewSQLiteFeaturesIndexerWithCallback(db sqlite.Database, to_index []sqlite.Table, cb sql_index.SQLiteIndexerFunc) (*sql_index.SQLiteIndexer, error) {
 	return sql_index.NewSQLiteIndexer(db, to_index, cb)
 }

vendor/github.com/whosonfirst/go-whosonfirst-index/index.go

@@ -38,7 +38,7 @@ github.com/whosonfirst/go-whosonfirst-geojson-v2/properties/whosonfirst
 github.com/whosonfirst/go-whosonfirst-geojson-v2/utils
 # github.com/whosonfirst/go-whosonfirst-hash v0.1.0
 github.com/whosonfirst/go-whosonfirst-hash
-# github.com/whosonfirst/go-whosonfirst-index v0.2.2
+# github.com/whosonfirst/go-whosonfirst-index v0.2.3
 github.com/whosonfirst/go-whosonfirst-index
 github.com/whosonfirst/go-whosonfirst-index/fs
 # github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972
@@ -62,7 +62,7 @@ github.com/whosonfirst/go-whosonfirst-spr
 github.com/whosonfirst/go-whosonfirst-sqlite
 github.com/whosonfirst/go-whosonfirst-sqlite/database
 github.com/whosonfirst/go-whosonfirst-sqlite/utils
-# github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.4
+# github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5
 github.com/whosonfirst/go-whosonfirst-sqlite-features/tables
 github.com/whosonfirst/go-whosonfirst-sqlite-features
 # github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.0