add SQLiteFeaturesIndexRelationsFuncWithOptions

Author: thisisaaronland

index.go

@@ -10,9 +10,9 @@ import (
 	"github.com/whosonfirst/go-whosonfirst-geojson-v2/feature"
 	wof_index "github.com/whosonfirst/go-whosonfirst-index"
 	"github.com/whosonfirst/go-whosonfirst-sqlite"
-	"github.com/whosonfirst/go-whosonfirst-uri"	
 	wof_tables "github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
 	sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index"
+	"github.com/whosonfirst/go-whosonfirst-uri"
 	"github.com/whosonfirst/warning"
 	"io"
 	"io/ioutil"
@@ -24,6 +24,11 @@ type SQLiteFeaturesLoadRecordFuncOptions struct {
 	StrictAltFiles bool
 }
 
+type SQLiteFeaturesIndexRelationsFuncOptions struct {
+	Reader reader.Reader
+	Strict bool
+}
+
 func SQLiteFeaturesLoadRecordFunc(opts *SQLiteFeaturesLoadRecordFuncOptions) sql_index.SQLiteIndexerLoadRecordFunc {
 
 	cb := func(ctx context.Context, fh io.Reader, args ...interface{}) (interface{}, error) {
@@ -82,8 +87,16 @@ func SQLiteFeaturesLoadRecordFunc(opts *SQLiteFeaturesLoadRecordFuncOptions) sql
 
 func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPostIndexFunc {
 
+	opts := &SQLiteFeaturesIndexRelationsFuncOptions{}
+	opts.Reader = r
+
+	return SQLiteFeaturesIndexRelationsFuncWithOptions(opts)
+}
+
+func SQLiteFeaturesIndexRelationsFuncWithOptions(opts *SQLiteFeaturesIndexRelationsFuncOptions) sql_index.SQLiteIndexerPostIndexFunc {
+
 	seen := new(sync.Map)
-	
+
 	cb := func(ctx context.Context, db sqlite.Database, tables []sqlite.Table, record interface{}) error {
 
 		geojson_t, err := wof_tables.NewGeoJSONTable()
@@ -137,7 +150,7 @@ func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPo
 			}
 
 			seen.Store(id, true)
-			
+
 			sql := fmt.Sprintf("SELECT COUNT(id) FROM %s WHERE id=?", geojson_t.Name())
 			row := conn.QueryRow(sql, id)
 
@@ -158,10 +171,16 @@ func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPo
 				return err
 			}
 
-			fh, err := r.Read(ctx, rel_path)
+			fh, err := opts.Reader.Read(ctx, rel_path)
 
-			if err != nil{
-				return err
+			if err != nil {
+
+				if opts.Strict {
+					return err
+				}
+
+				log.Printf("Failed to read '%s' because '%v'. Strict mode is disabled so skipping\n", rel_path, err)
+				continue
 			}
 
 			defer fh.Close()
@@ -170,9 +189,15 @@ func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPo
 
 			// check for warnings in case this record has a non-standard
 			// placetype (20201224/thisisaaronland)
-			
-			if err != nil && !warning.IsWarning(err){
-				return err
+
+			if err != nil && !warning.IsWarning(err) {
+
+				if opts.Strict {
+					return err
+				}
+
+				log.Printf("Failed to load feature for '%s' because '%v'. Strict mode is disabled so skipping\n", rel_path, err)
+				continue
 			}
 
 			to_index = append(to_index, ancestor)