remove whosonfirst/go-whosonfirst-reader; ignore placetype warning when loading features; add a previously seen lookup table when indexing relations

Author: thisisaaronland

go.mod

@@ -11,9 +11,10 @@ require (
 	github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972
 	github.com/whosonfirst/go-whosonfirst-index-sqlite v0.0.3
 	github.com/whosonfirst/go-whosonfirst-log v0.1.0
-	github.com/whosonfirst/go-whosonfirst-reader v0.0.2
+	github.com/whosonfirst/go-whosonfirst-reader v0.0.2 // indirect
 	github.com/whosonfirst/go-whosonfirst-sqlite v0.1.6
 	github.com/whosonfirst/go-whosonfirst-sqlite-features v0.6.4
 	github.com/whosonfirst/go-whosonfirst-sqlite-index v0.2.0
+	github.com/whosonfirst/go-whosonfirst-uri v0.2.0
 	github.com/whosonfirst/warning v0.1.1
 )

index.go

@@ -9,14 +9,15 @@ import (
 	"github.com/whosonfirst/go-whosonfirst-geojson-v2"
 	"github.com/whosonfirst/go-whosonfirst-geojson-v2/feature"
 	wof_index "github.com/whosonfirst/go-whosonfirst-index"
-	wof_reader "github.com/whosonfirst/go-whosonfirst-reader"
 	"github.com/whosonfirst/go-whosonfirst-sqlite"
+	"github.com/whosonfirst/go-whosonfirst-uri"	
 	wof_tables "github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
 	sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index"
 	"github.com/whosonfirst/warning"
 	"io"
 	"io/ioutil"
 	"log"
+	"sync"
 )
 
 type SQLiteFeaturesLoadRecordFuncOptions struct {
@@ -81,6 +82,8 @@ func SQLiteFeaturesLoadRecordFunc(opts *SQLiteFeaturesLoadRecordFuncOptions) sql
 
 func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPostIndexFunc {
 
+	seen := new(sync.Map)
+	
 	cb := func(ctx context.Context, db sqlite.Database, tables []sqlite.Table, record interface{}) error {
 
 		geojson_t, err := wof_tables.NewGeoJSONTable()
@@ -127,6 +130,14 @@ func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPo
 
 		for id, _ := range relations {
 
+			_, ok := seen.Load(id)
+
+			if ok {
+				continue
+			}
+
+			seen.Store(id, true)
+			
 			sql := fmt.Sprintf("SELECT COUNT(id) FROM %s WHERE id=?", geojson_t.Name())
 			row := conn.QueryRow(sql, id)
 
@@ -141,12 +152,29 @@ func SQLiteFeaturesIndexRelationsFunc(r reader.Reader) sql_index.SQLiteIndexerPo
 				continue
 			}
 
-			ancestor, err := wof_reader.LoadFeatureFromID(ctx, r, id)
+			rel_path, err := uri.Id2RelPath(id)
 
 			if err != nil {
 				return err
 			}
 
+			fh, err := r.Read(ctx, rel_path)
+
+			if err != nil{
+				return err
+			}
+
+			defer fh.Close()
+
+			ancestor, err := feature.LoadFeatureFromReader(fh)
+
+			// check for warnings in case this record has a non-standard
+			// placetype (20201224/thisisaaronland)
+			
+			if err != nil && !warning.IsWarning(err){
+				return err
+			}
+
 			to_index = append(to_index, ancestor)
 
 			// TO DO: CHECK WHETHER TO INDEX ALT FILES FOR ANCESTOR(S)

vendor/github.com/whosonfirst/go-whosonfirst-reader/.gitignore

@@ -61,8 +61,6 @@ github.com/whosonfirst/go-whosonfirst-names/tags
 # github.com/whosonfirst/go-whosonfirst-placetypes v0.1.0
 github.com/whosonfirst/go-whosonfirst-placetypes
 github.com/whosonfirst/go-whosonfirst-placetypes/placetypes
-# github.com/whosonfirst/go-whosonfirst-reader v0.0.2
-github.com/whosonfirst/go-whosonfirst-reader
 # github.com/whosonfirst/go-whosonfirst-sources v0.1.0
 github.com/whosonfirst/go-whosonfirst-sources
 github.com/whosonfirst/go-whosonfirst-sources/sources