Skip to content
This repository was archived by the owner on Dec 13, 2024. It is now read-only.

Commit 2cda2cc

Browse files
author
thisisaaronland
committed
wire in belongsto indexing, compiles but untested
1 parent b45c698 commit 2cda2cc

File tree

39 files changed

+1550
-43
lines changed

39 files changed

+1550
-43
lines changed

‎cmd/wof-sqlite-index-features/main.go

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
package main
22

33
import (
4-
"flag"
5-
"fmt"
6-
wof_index "github.com/whosonfirst/go-whosonfirst-index"
4+
_ "github.com/whosonfirst/go-reader-http"
75
_ "github.com/whosonfirst/go-whosonfirst-index-csv"
86
_ "github.com/whosonfirst/go-whosonfirst-index-sqlite"
97
_ "github.com/whosonfirst/go-whosonfirst-index/fs"
8+
)
9+
10+
import (
11+
"context"
12+
"flag"
13+
"fmt"
14+
"github.com/whosonfirst/go-reader"
15+
wof_index "github.com/whosonfirst/go-whosonfirst-index"
1016
log "github.com/whosonfirst/go-whosonfirst-log"
1117
"github.com/whosonfirst/go-whosonfirst-sqlite"
1218
"github.com/whosonfirst/go-whosonfirst-sqlite-features-index"
1319
"github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
20+
sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index"
1421
"github.com/whosonfirst/go-whosonfirst-sqlite/database"
1522
"io"
1623
"os"
@@ -43,6 +50,9 @@ func main() {
4350
alt_files := flag.Bool("index-alt-files", false, "Index alt geometries")
4451
strict_alt_files := flag.Bool("strict-alt-files", true, "Be strict when indexing alt geometries")
4552

53+
index_belongs_to := flag.Bool("index-belongs-to", false, "...")
54+
belongs_to_uri := flag.String("belongs-to-uri", "", "...")
55+
4656
var procs = flag.Int("processes", (runtime.NumCPU() * 2), "The number of concurrent processes to index data with")
4757

4858
flag.Parse()
@@ -200,12 +210,32 @@ func main() {
200210
logger.Fatal("You forgot to specify which (any) tables to index")
201211
}
202212

203-
opts := index.DefaultSQLiteFeaturesIndexerCallbackOptions()
204-
opts.StrictAltFiles = *strict_alt_files
213+
record_opts := &index.SQLiteFeaturesLoadRecordFuncOptions{
214+
StrictAltFiles: *strict_alt_files,
215+
}
216+
217+
record_func := index.SQLiteFeaturesLoadRecordFunc(record_opts)
218+
219+
idx_opts := &sql_index.SQLiteIndexerOptions{
220+
DB: db,
221+
Tables: to_index,
222+
LoadRecordFunc: record_func,
223+
}
224+
225+
if *index_belongs_to {
205226

206-
cb := index.SQLiteFeaturesIndexerCallback(opts)
227+
ctx := context.Background()
228+
r, err := reader.NewReader(ctx, *belongs_to_uri)
229+
230+
if err != nil {
231+
logger.Fatal("Failed to load reader (%s), %v", *belongs_to_uri, err)
232+
}
233+
234+
belongsto_func := index.SQLiteFeaturesIndexBelongsToFunc(r)
235+
idx_opts.PostIndexFunc = belongsto_func
236+
}
207237

208-
idx, err := index.NewSQLiteFeaturesIndexerWithCallback(db, to_index, cb)
238+
idx, err := sql_index.NewSQLiteIndexer(idx_opts)
209239

210240
if err != nil {
211241
logger.Fatal("failed to create sqlite indexer because %s", err)

‎go.mod

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@ module github.com/whosonfirst/go-whosonfirst-sqlite-features-index
33
go 1.12
44

55
require (
6-
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.0
6+
github.com/whosonfirst/go-reader v0.1.1
7+
github.com/whosonfirst/go-reader-http v0.0.2
8+
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.2
79
github.com/whosonfirst/go-whosonfirst-index v0.2.3
810
github.com/whosonfirst/go-whosonfirst-index-csv v0.0.0-20191002171239-c6712fe20972
911
github.com/whosonfirst/go-whosonfirst-index-sqlite v0.0.3
1012
github.com/whosonfirst/go-whosonfirst-log v0.1.0
13+
github.com/whosonfirst/go-whosonfirst-reader v0.0.1
1114
github.com/whosonfirst/go-whosonfirst-sqlite v0.1.5
12-
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.6
13-
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.1
15+
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.7
16+
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.2.0
1417
github.com/whosonfirst/warning v0.1.0
1518
)

‎go.sum

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
22
github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 h1:FSsoaa1q4jAaeiAUxf9H0PgFP7eA/UL6c3PdJH+nMN4=
33
github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718/go.mod h1:VVwKsx9Dc8rNG55BWqogoJzGubjKnRoXdUvpGbWqeCc=
44
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
5+
github.com/aaronland/go-roster v0.0.1 h1:r1l4n1HfWvEtOXZvhfXX0Won9Xf6QJsigdUdzOtuy/M=
6+
github.com/aaronland/go-roster v0.0.1/go.mod h1:AcovpxlG1XxJxX2Fjqlm63fEIBhCjEIBV4lP87FZDmI=
57
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
68
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
79
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -85,6 +87,11 @@ github.com/twpayne/go-geom v1.0.5 h1:XZBfc3Wx0dj4p17ZfmzqxnU9fTTa3pY4YG5RngKsVNI
8587
github.com/twpayne/go-geom v1.0.5/go.mod h1:gO3i8BeAvZuihwwXcw8dIOWXebCzTmy3uvXj9dZG2RA=
8688
github.com/twpayne/go-kml v1.0.0/go.mod h1:LlvLIQSfMqYk2O7Nx8vYAbSLv4K9rjMvLlEdUKWdjq0=
8789
github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
90+
github.com/whosonfirst/go-reader v0.0.2/go.mod h1:k/U2jXarXsFWC2yh63kYgKIJzTEmc60lN/ipyrdT3W4=
91+
github.com/whosonfirst/go-reader v0.1.1 h1:0fXoYQR/Yyh3PKk+9Huwoc6dCAt8l1tjg2dEShjqcFI=
92+
github.com/whosonfirst/go-reader v0.1.1/go.mod h1:w0AeU+Ojyfmxb7r+rcSYyxpCxrhxlHv+D9EgEwQsqcM=
93+
github.com/whosonfirst/go-reader-http v0.0.2 h1:uaCFfNu5Hwr5PDicduLAB1g3y6pi6fUcLJHUKlbNdOw=
94+
github.com/whosonfirst/go-reader-http v0.0.2/go.mod h1:u34jMGXs4smJyVHqGudWoTTy8u954gHqlz9cuIr7eBk=
8895
github.com/whosonfirst/go-rfc-5646 v0.1.0 h1:HNFPAem6v5De61PXLgbGzx9tfNOP83AAkVvm9WAddJY=
8996
github.com/whosonfirst/go-rfc-5646 v0.1.0/go.mod h1:JZj//FV9YeV3fkyOY/82V53EMLQXwRwNPuQIGs8BUmo=
9097
github.com/whosonfirst/go-spatialite v0.0.0-20180220171945-cb1d9ed624a8 h1:5dtYgPS6su1QqwUfVnyEL9V7GvMWX+60VRkUX5RvJsc=
@@ -106,6 +113,9 @@ github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.11.1 h1:WKIZkzlOBgfE6AuVT8xy
106113
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.11.1/go.mod h1:uriIqN/EQVo1QllqvpIcAyshnEOggPlraUYjjeyGTBg=
107114
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.0 h1:aYsgIq/g4fU5rVdumMTqG9ysJxwhW085buzaMxP5mjs=
108115
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.0/go.mod h1:hC145O2TiPTU8zemHRriKOkG972BNwEowqjxqHeWDwM=
116+
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.1/go.mod h1:hC145O2TiPTU8zemHRriKOkG972BNwEowqjxqHeWDwM=
117+
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.2 h1:SRwpX4y+qGu9BmNjezhXqOfV0MEVv5o2rRnbcoAjvPU=
118+
github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.12.2/go.mod h1:hC145O2TiPTU8zemHRriKOkG972BNwEowqjxqHeWDwM=
109119
github.com/whosonfirst/go-whosonfirst-hash v0.1.0 h1:FpnclPIb+8M1uhSXfl3z8nYcG/3O59vgfkdV+m0hQpA=
110120
github.com/whosonfirst/go-whosonfirst-hash v0.1.0/go.mod h1:1ZdCFZTnQt5bwnsj2daB9yHilKOKToVh+Tyj/Z8TbUk=
111121
github.com/whosonfirst/go-whosonfirst-index v0.1.1 h1:AV2dVzt0F9pAupbpsl4TpBxZLeOqL4jtKZKb1gAoJT0=
@@ -128,6 +138,9 @@ github.com/whosonfirst/go-whosonfirst-names v0.1.0 h1:uXop/DwQqH60uDBZvHCPg1yRSQ
128138
github.com/whosonfirst/go-whosonfirst-names v0.1.0/go.mod h1:0z86/nedM9T/5C8cAdbCMfRuBrkc33oEQ6vdJ6WybSg=
129139
github.com/whosonfirst/go-whosonfirst-placetypes v0.1.0 h1:zuSk8eqeEkg42sIZ4EF71IMtphdTbG80qJsXhuZXXbM=
130140
github.com/whosonfirst/go-whosonfirst-placetypes v0.1.0/go.mod h1:Jdmug2QQLbrmg+UcYGz8k575GnrOEg63vZVS46e5fMs=
141+
github.com/whosonfirst/go-whosonfirst-reader v0.0.0-20191122203133-047f20452865/go.mod h1:xYzBn6llLD/uS6Zhh0H4LAqddEJHXdQSUIxywTnhpOU=
142+
github.com/whosonfirst/go-whosonfirst-reader v0.0.1 h1:8OC0/iOtElJXg6W/P3mROuNK2sZqQBDtvMN30sKfq5U=
143+
github.com/whosonfirst/go-whosonfirst-reader v0.0.1/go.mod h1:SNGLXBF9h8mmf2S3CYKPBtO9d4D8n1TohSKSmoRmvqo=
131144
github.com/whosonfirst/go-whosonfirst-sources v0.1.0 h1:JuKLa6KWke22jBfJ1pM9WQHoz1/3pbDv2C+aR+THPPQ=
132145
github.com/whosonfirst/go-whosonfirst-sources v0.1.0/go.mod h1:EUMHyGzUmqPPxlMmOp+28BFeoBdxxE0HCKRd67lkqGM=
133146
github.com/whosonfirst/go-whosonfirst-spr v0.1.0 h1:5qE629nCiucF2upy5NjPOEl9cFatsljykYY0l2JKgAk=
@@ -155,12 +168,16 @@ github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5 h1:sicGwGk1YK26M9P+
155168
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.5/go.mod h1:jXNvI/rNqrL9nXkTefSBChmepQuOn9YMb/xK1LDO+jA=
156169
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.6 h1:uPvnvC3bnmVWxu8z9fCdh/uU2mf/xU5ze6t4sHWjlYs=
157170
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.6/go.mod h1:C+ARiF6KebUj77c+pAawXtMf5028285Qg50FHUj5ou8=
171+
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.7 h1:S3H+n72XkWrLyK33IFEwnKb0yoy+xe0QIUBiOwA/bKw=
172+
github.com/whosonfirst/go-whosonfirst-sqlite-features v0.2.7/go.mod h1:Bhtd9O7wjm9k4NMaoGErUdcbNRR0DstKD+ImtXRbc5U=
158173
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.0.1 h1:/GuAgxIHn5vZwAEDA4XUSWe7rYWZW30I4HLQRTpBeJY=
159174
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.0.1/go.mod h1:5fY1ikCDhe1u0umawonTOICPzv/2CgfFT9hTAen5nsI=
160175
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.0 h1:/tljUy2rchRC9XE4FDteNmYZYhYqxWX2uF4c61WlocU=
161176
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.0/go.mod h1:Roi0glm3gVxDq+i32dUxGWTWxb+80nPanz/UrVatmwk=
162177
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.1 h1:UNHdFAr1zUfDx00NoPtzaIWJUdbwV5U8WzhqtdhAmc4=
163178
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.1.1/go.mod h1:NzDYzzrFMqRpDVkh0DUDs3WfdHywsfnInycY5JTj3zc=
179+
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.2.0 h1:biGKWUogPLXmWVOn44Jrb1FcE2fCVU3sV4w3sLYF1Yw=
180+
github.com/whosonfirst/go-whosonfirst-sqlite-index v0.2.0/go.mod h1:Z5lr3lHDAmMkexXXjFGPP4Cw356ULmsiyv0Tok6R86k=
164181
github.com/whosonfirst/go-whosonfirst-uri v0.1.0 h1:JMlpam0x1hVrFBMTAPY3edIHz7azfMK8lLI2kM9BgbI=
165182
github.com/whosonfirst/go-whosonfirst-uri v0.1.0/go.mod h1:8eaDVcc4v+HHHEDaRbApdmhPwM4/JQllw2PktvZcPVs=
166183
github.com/whosonfirst/walk v0.0.0-20160802000000-c0a349674b73681a7272f5ce6ade8ea28055059f h1:hvKIIx2IuWmRtOdpDk29quD+t7GowpHZxz8bCfIGE58=

‎index.go

Lines changed: 65 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,26 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"github.com/whosonfirst/go-reader"
8+
"github.com/whosonfirst/go-whosonfirst-geojson-v2"
79
"github.com/whosonfirst/go-whosonfirst-geojson-v2/feature"
10+
"github.com/whosonfirst/go-whosonfirst-geojson-v2/properties/whosonfirst"
811
wof_index "github.com/whosonfirst/go-whosonfirst-index"
12+
wof_reader "github.com/whosonfirst/go-whosonfirst-reader"
913
"github.com/whosonfirst/go-whosonfirst-sqlite"
14+
wof_tables "github.com/whosonfirst/go-whosonfirst-sqlite-features/tables"
1015
sql_index "github.com/whosonfirst/go-whosonfirst-sqlite-index"
1116
"github.com/whosonfirst/warning"
1217
"io"
1318
"io/ioutil"
1419
"log"
1520
)
1621

17-
type SQLiteFeaturesIndexerCallbackOptions struct {
22+
type SQLiteFeaturesLoadRecordFuncOptions struct {
1823
StrictAltFiles bool
1924
}
2025

21-
func DefaultSQLiteFeaturesIndexerCallbackOptions() *SQLiteFeaturesIndexerCallbackOptions {
22-
23-
opts := &SQLiteFeaturesIndexerCallbackOptions{
24-
StrictAltFiles: true,
25-
}
26-
27-
return opts
28-
}
29-
30-
func SQLiteFeaturesIndexerCallback(opts *SQLiteFeaturesIndexerCallbackOptions) sql_index.SQLiteIndexerFunc {
26+
func SQLiteFeaturesLoadRecordFunc(opts *SQLiteFeaturesLoadRecordFuncOptions) sql_index.SQLiteIndexerLoadRecordFunc {
3127

3228
cb := func(ctx context.Context, fh io.Reader, args ...interface{}) (interface{}, error) {
3329

@@ -83,14 +79,65 @@ func SQLiteFeaturesIndexerCallback(opts *SQLiteFeaturesIndexerCallbackOptions) s
8379
return cb
8480
}
8581

86-
func NewDefaultSQLiteFeaturesIndexer(db sqlite.Database, to_index []sqlite.Table) (*sql_index.SQLiteIndexer, error) {
82+
func SQLiteFeaturesIndexBelongsToFunc(r reader.Reader) sql_index.SQLiteIndexerPostIndexFunc {
8783

88-
opts := DefaultSQLiteFeaturesIndexerCallbackOptions()
89-
cb := SQLiteFeaturesIndexerCallback(opts)
84+
cb := func(ctx context.Context, db sqlite.Database, tables []sqlite.Table, record interface{}) error {
9085

91-
return NewSQLiteFeaturesIndexerWithCallback(db, to_index, cb)
92-
}
86+
geojson_t, err := wof_tables.NewGeoJSONTable()
87+
88+
if err != nil {
89+
return err
90+
}
91+
92+
conn, err := db.Conn()
93+
94+
if err != nil {
95+
return err
96+
}
97+
98+
f := record.(geojson.Feature)
99+
belongsto := whosonfirst.BelongsTo(f)
100+
101+
to_index := make([]geojson.Feature, 0)
102+
103+
for _, id := range belongsto {
104+
105+
sql := fmt.Sprintf("SELECT COUNT(id) FROM %s WHERE id=?", geojson_t.Name())
106+
row := conn.QueryRow(sql, id)
107+
108+
var count int
109+
err = row.Scan(&count)
110+
111+
if err != nil {
112+
return err
113+
}
114+
115+
if count == 0 {
116+
117+
ancestor, err := wof_reader.LoadFeatureFromID(ctx, r, id)
118+
119+
if err != nil {
120+
return err
121+
}
122+
123+
to_index = append(to_index, ancestor)
124+
}
125+
}
126+
127+
for _, record := range to_index {
93128

94-
func NewSQLiteFeaturesIndexerWithCallback(db sqlite.Database, to_index []sqlite.Table, cb sql_index.SQLiteIndexerFunc) (*sql_index.SQLiteIndexer, error) {
95-
return sql_index.NewSQLiteIndexer(db, to_index, cb)
129+
for _, t := range tables {
130+
131+
err = t.IndexRecord(db, record)
132+
133+
if err != nil {
134+
return err
135+
}
136+
}
137+
}
138+
139+
return nil
140+
}
141+
142+
return cb
96143
}

‎vendor/github.com/aaronland/go-roster/.gitignore

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/aaronland/go-roster/LICENSE

Lines changed: 27 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/aaronland/go-roster/README.md

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/aaronland/go-roster/default.go

Lines changed: 92 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/aaronland/go-roster/go.mod

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)