Issue 19: Deprecate -index-alt-files=bool flag, replace with (zero or more) -index_alt=table flags (#20)

* snapshot: start working through issue #19

* add -spelunker-tables flag

---------

Co-authored-by: thisisaaronland <devnull@localhost>
Co-authored-by: thisisaaronland <thisisaaronland@localhost>

Author: 3 people

Makefile

@@ -9,3 +9,15 @@ cli:
 		-o bin/wof-sqlite-index-features-mattn \
 		-tags "icu json1 fts5" \
 		cmd/wof-sqlite-index-features-mattn/main.go
+
+debug:
+	./bin/wof-sqlite-index-features-mattn \
+		-timings \
+		-database-uri mattn:///usr/local/data/ca-3.db \
+		-rtree \
+		-spr \
+		-geojson \
+		-concordances \
+		-search \
+		-index-alt geojson \
+		/usr/local/data/whosonfirst-data-admin-ca

README.md

@@ -33,8 +33,10 @@ $> ./bin/wof-sqlite-index-features -h
     	Index the 'geojson' table
   -geometries
     	Index the 'geometries' table (requires that libspatialite already be installed)
+  -index-alt value
+    	Zero or more table names where alt geometry files should be indexed.
   -index-alt-files
-    	Index alt geometries
+    	Index alt geometries. This flag is deprecated, please use -index-alt=TABLE,TABLE,etc. instead. To index alt geometries in all the applicable tables use -index-alt=*
   -index-relations
     	Index the records related to a feature, specifically wof:belongsto, wof:depicts and wof:involves. Alt files for relations are not indexed at this time.
   -index-relations-reader-uri string
@@ -48,7 +50,7 @@ $> ./bin/wof-sqlite-index-features -h
   -optimize
     	Attempt to optimize the database before closing connection (default true)
   -processes int
-    	The number of concurrent processes to index data with (default 8)
+    	The number of concurrent processes to index data with (default 16)
   -properties
     	Index the 'properties' table
   -rtree
@@ -57,6 +59,8 @@ $> ./bin/wof-sqlite-index-features -h
     	Index the 'search' table (using SQLite FTS4 full-text indexer)
   -spatial-tables
     	If true then index the necessary tables for use with the whosonfirst/go-whosonfirst-spatial-sqlite package.
+  -spelunker-tables
+    	If true then index the necessary tables for use with the whosonfirst/go-whosonfirst-spelunker packages
   -spr
     	Index the 'spr' table
   -strict-alt-files

app/index/app.go

@@ -6,20 +6,26 @@ import (
 	"fmt"
 	"log"
 	"runtime"
+	"slices"
 
 	"github.com/aaronland/go-sqlite/v2"
 	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/whosonfirst/go-reader"
+	sql_tables "github.com/whosonfirst/go-whosonfirst-sql/tables"
 	"github.com/whosonfirst/go-whosonfirst-sqlite-features-index/v2"
 	"github.com/whosonfirst/go-whosonfirst-sqlite-features/v2/tables"
 	sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index/v4"
 )
 
+const index_alt_all string = "*"
+
 func Run(ctx context.Context, logger *log.Logger) error {
 	fs := DefaultFlagSet()
 	return RunWithFlagSet(ctx, fs, logger)
 }
 
+// To do: Add RunWithOptions...
+
 func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) error {
 
 	flagset.Parse(fs)
@@ -33,6 +39,15 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		spr = true
 	}
 
+	if spelunker_tables {
+		rtree = true
+		spr = true
+		geojson = true
+		concordances = true
+		ancestors = true
+		search = true
+	}
+	
 	db, err := sqlite.NewDatabase(ctx, db_uri)
 
 	if err != nil {
@@ -63,7 +78,6 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		}()
 
 	} else {
-
 		defer db.Close(ctx)
 	}
 
@@ -83,15 +97,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		geojson_opts, err := tables.DefaultGeoJSONTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geojson' table options because %s", err)
+			return fmt.Errorf("failed to create '%s' table options because %s", sql_tables.GEOJSON_TABLE_NAME, err)
 		}
 
-		geojson_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.GEOJSON_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			geojson_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewGeoJSONTableWithDatabaseAndOptions(ctx, db, geojson_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geojson' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.GEOJSON_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, gt)
@@ -102,7 +120,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		t, err := tables.NewSupersedesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'supersedes' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.SUPERSEDES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, t)
@@ -116,7 +134,11 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 			return fmt.Errorf("failed to create 'rtree' table options because %s", err)
 		}
 
-		rtree_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.RTREE_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			rtree_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewRTreeTableWithDatabaseAndOptions(ctx, db, rtree_opts)
 
@@ -135,7 +157,11 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 			return fmt.Errorf("failed to create 'properties' table options because %s", err)
 		}
 
-		properties_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.PROPERTIES_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			properties_opts.IndexAltFiles = true
+		}
 
 		gt, err := tables.NewPropertiesTableWithDatabaseAndOptions(ctx, db, properties_opts)
 
@@ -151,15 +177,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		spr_opts, err := tables.DefaultSPRTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("Failed to create 'spr' table options because %v", err)
+			return fmt.Errorf("Failed to create '%s' table options because %v", sql_tables.SPR_TABLE_NAME, err)
 		}
 
-		spr_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.SPR_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			spr_opts.IndexAltFiles = true
+		}
 
 		st, err := tables.NewSPRTableWithDatabaseAndOptions(ctx, db, spr_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'spr' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.SPR_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, st)
@@ -170,7 +200,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		nm, err := tables.NewNamesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'names' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.NAMES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, nm)
@@ -181,7 +211,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		an, err := tables.NewAncestorsTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'ancestors' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.ANCESTORS_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, an)
@@ -192,7 +222,7 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		cn, err := tables.NewConcordancesTableWithDatabase(ctx, db)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'concordances' table because %s", err)
+			return fmt.Errorf("failed to create '%s' table because %s", sql_tables.CONCORDANCES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, cn)
@@ -206,15 +236,19 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 		geometries_opts, err := tables.DefaultGeometriesTableOptions()
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geometries' table options because %v", err)
+			return fmt.Errorf("failed to create '%s' table options because %v", sql_tables.GEOMETRIES_TABLE_NAME, err)
 		}
 
-		geometries_opts.IndexAltFiles = alt_files
+		// alt_files is deprecated (20240229/straup)
+
+		if alt_files || slices.Contains(index_alt, sql_tables.CONCORDANCES_TABLE_NAME) || slices.Contains(index_alt, index_alt_all) {
+			geometries_opts.IndexAltFiles = true
+		}
 
 		gm, err := tables.NewGeometriesTableWithDatabaseAndOptions(ctx, db, geometries_opts)
 
 		if err != nil {
-			return fmt.Errorf("failed to create 'geometries' table because %v", err)
+			return fmt.Errorf("failed to create '%s' table because %v", sql_tables.CONCORDANCES_TABLE_NAME, err)
 		}
 
 		to_index = append(to_index, gm)
@@ -226,6 +260,8 @@ func RunWithFlagSet(ctx context.Context, fs *flag.FlagSet, logger *log.Logger) e
 
 	if search {
 
+		// ALT FILES...
+
 		st, err := tables.NewSearchTableWithDatabase(ctx, db)
 
 		if err != nil {

app/index/flags.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/sfomuseum/go-flags/flagset"
+	"github.com/sfomuseum/go-flags/multi"
 	"github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter"
 )
 
@@ -25,7 +26,9 @@ var properties bool
 var search bool
 var spr bool
 var supersedes bool
+
 var spatial_tables bool
+var spelunker_tables bool
 
 var live_hard bool
 var timings bool
@@ -34,6 +37,8 @@ var optimize bool
 var alt_files bool
 var strict_alt_files bool
 
+var index_alt multi.MultiString
+
 var index_relations bool
 var relations_uri string
 
@@ -63,12 +68,15 @@ func DefaultFlagSet() *flag.FlagSet {
 	fs.BoolVar(&supersedes, "supersedes", false, "Index the 'supersedes' table")
 
 	fs.BoolVar(&spatial_tables, "spatial-tables", false, "If true then index the necessary tables for use with the whosonfirst/go-whosonfirst-spatial-sqlite package.")
+	fs.BoolVar(&spelunker_tables, "spelunker-tables", false, "If true then index the necessary tables for use with the whosonfirst/go-whosonfirst-spelunker packages")	
 
 	fs.BoolVar(&live_hard, "live-hard-die-fast", true, "Enable various performance-related pragmas at the expense of possible (unlikely) database corruption")
 	fs.BoolVar(&timings, "timings", false, "Display timings during and after indexing")
 	fs.BoolVar(&optimize, "optimize", true, "Attempt to optimize the database before closing connection")
 
-	fs.BoolVar(&alt_files, "index-alt-files", false, "Index alt geometries")
+	fs.BoolVar(&alt_files, "index-alt-files", false, "Index alt geometries. This flag is deprecated, please use -index-alt=TABLE,TABLE,etc. instead. To index alt geometries in all the applicable tables use -index-alt=*")
+	fs.Var(&index_alt, "index-alt", "Zero or more table names where alt geometry files should be indexed.")
+
 	fs.BoolVar(&strict_alt_files, "strict-alt-files", true, "Be strict when indexing alt geometries")
 
 	fs.BoolVar(&index_relations, "index-relations", false, "Index the records related to a feature, specifically wof:belongsto, wof:depicts and wof:involves. Alt files for relations are not indexed at this time.")

go.mod

@@ -1,19 +1,20 @@
 module github.com/whosonfirst/go-whosonfirst-sqlite-features-index/v2
 
-go 1.21.3
+go 1.22
 
 require (
 	github.com/aaronland/go-sqlite-mattn v0.0.3
 	github.com/aaronland/go-sqlite-modernc v0.0.3
 	github.com/aaronland/go-sqlite/v2 v2.2.0
 	github.com/sfomuseum/go-flags v0.10.0
-	github.com/tidwall/gjson v1.17.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/whosonfirst/go-reader v1.0.2
 	github.com/whosonfirst/go-reader-http v0.3.1
 	github.com/whosonfirst/go-whosonfirst-feature v0.0.27
 	github.com/whosonfirst/go-whosonfirst-iterate-git/v2 v2.1.4
 	github.com/whosonfirst/go-whosonfirst-iterate/v2 v2.3.4
-	github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2
+	github.com/whosonfirst/go-whosonfirst-sql v0.0.3
+	github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3
 	github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0
 	github.com/whosonfirst/go-whosonfirst-uri v1.3.0
 )
@@ -55,7 +56,6 @@ require (
 	github.com/whosonfirst/go-whosonfirst-names v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-sources v0.1.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7 // indirect
-	github.com/whosonfirst/go-whosonfirst-sql v0.0.2 // indirect
 	github.com/whosonfirst/walk v0.0.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.mongodb.org/mongo-driver v1.11.4 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -143,10 +143,10 @@ github.com/whosonfirst/go-whosonfirst-sources v0.1.0 h1:JuKLa6KWke22jBfJ1pM9WQHo
 github.com/whosonfirst/go-whosonfirst-sources v0.1.0/go.mod h1:EUMHyGzUmqPPxlMmOp+28BFeoBdxxE0HCKRd67lkqGM=
 github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7 h1:1j4IMuVua4/NhqKm2ke16h2V5Z2XiLoOdXbjDrfZb0E=
 github.com/whosonfirst/go-whosonfirst-spr/v2 v2.3.7/go.mod h1:I2IqsIutlDrvYp37Xzwk4WGq+m/gLh+gEifiLmH0bac=
-github.com/whosonfirst/go-whosonfirst-sql v0.0.2 h1:YkGkDxgeZiu4lhWFu4I/2f9uczgmwpW4wNV67xoRKAo=
-github.com/whosonfirst/go-whosonfirst-sql v0.0.2/go.mod h1:sQXkJTPfnxXbOPTCCxGjUydfQqATfEyO4xyxj2pEX3E=
-github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2 h1:ppfP8QKV6LMxnTeHgGE2ObJ+Txh99cHXjakDi0ohqHQ=
-github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.2/go.mod h1:8QuXkCVjJ1wrCESwF2Rr3PvN/4yD6Zkw4QWLSdEesmg=
+github.com/whosonfirst/go-whosonfirst-sql v0.0.3 h1:yyAdKO8bjuhnFsV0HHeAprd7HExpZo54/7K2+10kHSg=
+github.com/whosonfirst/go-whosonfirst-sql v0.0.3/go.mod h1:sQXkJTPfnxXbOPTCCxGjUydfQqATfEyO4xyxj2pEX3E=
+github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3 h1:tDt4MkomZKljI7Q8IYbB7L2XfKy69Y73/qJL/BzbzyY=
+github.com/whosonfirst/go-whosonfirst-sqlite-features/v2 v2.0.3/go.mod h1:TUDGyZW2nIWZ4gB7nJ0eTt0+p8izYXhyhoiOf1fOFFY=
 github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0 h1:JEDFIFN7bgQJTDAE1Bc0dYbnWUKF4M8Vsjr4AxsqI/A=
 github.com/whosonfirst/go-whosonfirst-sqlite-index/v4 v4.0.0/go.mod h1:ZM85QUfMKGRywkWmsT3SfmzMG2vKEzPmQdj3II08IGU=
 github.com/whosonfirst/go-whosonfirst-uri v1.3.0 h1:LYOVLqP9rWQxauYVkdw65j5LZxEi8OK0GHh/qCEpX4g=

index.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
+	"log/slog"
 	"sync"
 
 	_ "github.com/aaronland/go-sqlite-modernc"
@@ -175,7 +175,7 @@ func SQLiteFeaturesIndexRelationsFuncWithOptions(opts *SQLiteFeaturesIndexRelati
 					return fmt.Errorf("Failed to open %s, %v", rel_path, err)
 				}
 
-				log.Printf("Failed to read '%s' because '%v'. Strict mode is disabled so skipping\n", rel_path, err)
+				slog.Debug("Failed to read '%s' because '%v'. Strict mode is disabled so skipping\n", rel_path, err)
 				continue
 			}