diff --git a/src/NRedisStack/Search/Literals/AttributeOptions.cs b/src/NRedisStack/Search/Literals/FieldOptions.cs similarity index 64% rename from src/NRedisStack/Search/Literals/AttributeOptions.cs rename to src/NRedisStack/Search/Literals/FieldOptions.cs index 41a13714..25f623cc 100644 --- a/src/NRedisStack/Search/Literals/AttributeOptions.cs +++ b/src/NRedisStack/Search/Literals/FieldOptions.cs @@ -1,11 +1,13 @@ namespace NRedisStack.Search.Literals; -internal class AttributeOptions +internal class FieldOptions { public const string SORTABLE = "SORTABLE"; public const string UNF = "UNF"; public const string NOSTEM = "NOSTEM"; public const string NOINDEX = "NOINDEX"; + public const string INDEXEMPTY = "INDEXEMPTY"; + public const string INDEXMISSING = "INDEXMISSING"; //TODO: add all options } \ No newline at end of file diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs index b0d379ee..9822673e 100644 --- a/src/NRedisStack/Search/Schema.cs +++ b/src/NRedisStack/Search/Schema.cs @@ -62,10 +62,12 @@ public class TextField : Field public bool Unf { get; } public bool NoIndex { get; } public bool WithSuffixTrie { get; } + public bool MissingIndex { get; } + public bool EmptyIndex { get; } public TextField(FieldName name, double weight = 1.0, bool noStem = false, string? phonetic = null, bool sortable = false, bool unf = false, - bool noIndex = false, bool withSuffixTrie = false) + bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) : base(name, FieldType.Text) { Weight = weight; @@ -79,12 +81,14 @@ public TextField(FieldName name, double weight = 1.0, bool noStem = false, Unf = unf; NoIndex = noIndex; WithSuffixTrie = withSuffixTrie; + MissingIndex = missingIndex; + EmptyIndex = emptyIndex; } public TextField(string name, double weight = 1.0, bool noStem = false, string? phonetic = null, bool sortable = false, bool unf = false, - bool noIndex = false, bool withSuffixTrie = false) - : this(FieldName.Of(name), weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie) { } + bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) + : this(FieldName.Of(name), weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie, missingIndex, emptyIndex) { } internal override void AddFieldTypeArgs(List args) { @@ -93,8 +97,11 @@ internal override void AddFieldTypeArgs(List args) AddPhonetic(args); AddWeight(args); if (WithSuffixTrie) args.Add(SearchArgs.WITHSUFFIXTRIE); - if (Sortable) args.Add(AttributeOptions.SORTABLE); + if (Sortable) args.Add(FieldOptions.SORTABLE); if (Unf) args.Add(SearchArgs.UNF); + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); + if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY); + } private void AddWeight(List args) @@ -124,10 +131,12 @@ public class TagField : Field public string Separator { get; } public bool CaseSensitive { get; } public bool WithSuffixTrie { get; } + public bool MissingIndex { get; } + public bool EmptyIndex { get; } internal TagField(FieldName name, bool sortable = false, bool unf = false, bool noIndex = false, string separator = ",", - bool caseSensitive = false, bool withSuffixTrie = false) + bool caseSensitive = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) : base(name, FieldType.Tag) { Sortable = sortable; @@ -136,12 +145,14 @@ internal TagField(FieldName name, bool sortable = false, bool unf = false, Separator = separator; CaseSensitive = caseSensitive; WithSuffixTrie = withSuffixTrie; + EmptyIndex = emptyIndex; + MissingIndex = missingIndex; } internal TagField(string name, bool sortable = false, bool unf = false, bool noIndex = false, string separator = ",", - bool caseSensitive = false, bool withSuffixTrie = false) - : this(FieldName.Of(name), sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie) { } + bool caseSensitive = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) + : this(FieldName.Of(name), sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie, missingIndex, emptyIndex) { } internal override void AddFieldTypeArgs(List args) { @@ -154,8 +165,10 @@ internal override void AddFieldTypeArgs(List args) args.Add(Separator); } if (CaseSensitive) args.Add(SearchArgs.CASESENSITIVE); - if (Sortable) args.Add(AttributeOptions.SORTABLE); + if (Sortable) args.Add(FieldOptions.SORTABLE); if (Unf) args.Add(SearchArgs.UNF); + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); + if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY); } } @@ -163,20 +176,24 @@ public class GeoField : Field { public bool Sortable { get; } public bool NoIndex { get; } - internal GeoField(FieldName name, bool sortable = false, bool noIndex = false) + public bool MissingIndex { get; } + + internal GeoField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false) : base(name, FieldType.Geo) { Sortable = sortable; NoIndex = noIndex; + MissingIndex = missingIndex; } - internal GeoField(string name, bool sortable = false, bool noIndex = false) - : this(FieldName.Of(name), sortable, noIndex) { } + internal GeoField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false) + : this(FieldName.Of(name), sortable, noIndex, missingIndex) { } internal override void AddFieldTypeArgs(List args) { if (NoIndex) args.Add(SearchArgs.NOINDEX); - if (Sortable) args.Add(AttributeOptions.SORTABLE); + if (Sortable) args.Add(FieldOptions.SORTABLE); + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); } } @@ -196,19 +213,22 @@ public enum CoordinateSystem SPHERICAL } private CoordinateSystem system { get; } + public bool MissingIndex { get; } - internal GeoShapeField(FieldName name, CoordinateSystem system) + internal GeoShapeField(FieldName name, CoordinateSystem system, bool missingIndex = false) : base(name, FieldType.GeoShape) { this.system = system; + MissingIndex = missingIndex; } - internal GeoShapeField(string name, CoordinateSystem system) - : this(FieldName.Of(name), system) { } + internal GeoShapeField(string name, CoordinateSystem system, bool missingIndex = false) + : this(FieldName.Of(name), system, missingIndex) { } internal override void AddFieldTypeArgs(List args) { args.Add(system.ToString()); + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); } } @@ -216,20 +236,24 @@ public class NumericField : Field { public bool Sortable { get; } public bool NoIndex { get; } - internal NumericField(FieldName name, bool sortable = false, bool noIndex = false) + public bool MissingIndex { get; } + + internal NumericField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false) : base(name, FieldType.Numeric) { Sortable = sortable; NoIndex = noIndex; + MissingIndex = missingIndex; } - internal NumericField(string name, bool sortable = false, bool noIndex = false) - : this(FieldName.Of(name), sortable, noIndex) { } + internal NumericField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false) + : this(FieldName.Of(name), sortable, noIndex, missingIndex) { } internal override void AddFieldTypeArgs(List args) { if (NoIndex) args.Add(SearchArgs.NOINDEX); - if (Sortable) args.Add(AttributeOptions.SORTABLE); + if (Sortable) args.Add(FieldOptions.SORTABLE); + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); } } @@ -244,15 +268,18 @@ public enum VectorAlgo public VectorAlgo Algorithm { get; } public Dictionary? Attributes { get; } - public VectorField(FieldName name, VectorAlgo algorithm, Dictionary? attributes = null) + public bool MissingIndex { get; } + + public VectorField(FieldName name, VectorAlgo algorithm, Dictionary? attributes = null, bool missingIndex = false) : base(name, FieldType.Vector) { Algorithm = algorithm; Attributes = attributes; + MissingIndex = missingIndex; } - public VectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null) - : this(FieldName.Of(name), algorithm, attributes) { } + public VectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null, bool missingIndex = false) + : this(FieldName.Of(name), algorithm, attributes, missingIndex) { } internal override void AddFieldTypeArgs(List args) { @@ -267,6 +294,7 @@ internal override void AddFieldTypeArgs(List args) args.Add(attribute.Value); } } + if (MissingIndex) args.Add(FieldOptions.INDEXMISSING); } } public List Fields { get; } = new List(); @@ -296,9 +324,9 @@ public Schema AddField(Field field) /// Keeps a suffix trie with all terms which match the suffix. /// The object. public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false, - string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false) + string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) { - Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie)); + Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie, missingIndex, emptyIndex)); return this; } @@ -316,9 +344,9 @@ public Schema AddTextField(string name, double weight = 1.0, bool sortable = fal /// Keeps a suffix trie with all terms which match the suffix. /// The object. public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false, - string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false) + string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) { - Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie)); + Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie, missingIndex, emptyIndex)); return this; } @@ -328,9 +356,9 @@ public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = /// The field's name. /// The coordinate system to use. /// The object. - public Schema AddGeoShapeField(string name, CoordinateSystem system) + public Schema AddGeoShapeField(string name, CoordinateSystem system, bool missingIndex = false) { - Fields.Add(new GeoShapeField(name, system)); + Fields.Add(new GeoShapeField(name, system, missingIndex)); return this; } @@ -340,9 +368,9 @@ public Schema AddGeoShapeField(string name, CoordinateSystem system) /// The field's name. /// The coordinate system to use. /// The object. - public Schema AddGeoShapeField(FieldName name, CoordinateSystem system) + public Schema AddGeoShapeField(FieldName name, CoordinateSystem system, bool missingIndex = false) { - Fields.Add(new GeoShapeField(name, system)); + Fields.Add(new GeoShapeField(name, system, missingIndex)); return this; } @@ -353,9 +381,9 @@ public Schema AddGeoShapeField(FieldName name, CoordinateSystem system) /// If true, the text field can be sorted. /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = false) + public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false) { - Fields.Add(new GeoField(name, sortable, noIndex)); + Fields.Add(new GeoField(name, sortable, noIndex, missingIndex)); return this; } @@ -366,9 +394,9 @@ public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = /// If true, the text field can be sorted. /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddGeoField(string name, bool sortable = false, bool noIndex = false) + public Schema AddGeoField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false) { - Fields.Add(new GeoField(name, sortable, noIndex)); + Fields.Add(new GeoField(name, sortable, noIndex, missingIndex)); return this; } @@ -379,9 +407,9 @@ public Schema AddGeoField(string name, bool sortable = false, bool noIndex = fal /// If true, the text field can be sorted. /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddNumericField(FieldName name, bool sortable = false, bool noIndex = false) + public Schema AddNumericField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false) { - Fields.Add(new NumericField(name, sortable, noIndex)); + Fields.Add(new NumericField(name, sortable, noIndex, missingIndex)); return this; } @@ -392,9 +420,9 @@ public Schema AddNumericField(FieldName name, bool sortable = false, bool noInde /// If true, the text field can be sorted. /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddNumericField(string name, bool sortable = false, bool noIndex = false) + public Schema AddNumericField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false) { - Fields.Add(new NumericField(name, sortable, noIndex)); + Fields.Add(new NumericField(name, sortable, noIndex, missingIndex)); return this; } @@ -412,9 +440,9 @@ public Schema AddNumericField(string name, bool sortable = false, bool noIndex = /// The object. public Schema AddTagField(FieldName name, bool sortable = false, bool unf = false, bool noIndex = false, string separator = ",", - bool caseSensitive = false, bool withSuffixTrie = false) + bool caseSensitive = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) { - Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie)); + Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie, missingIndex, emptyIndex)); return this; } @@ -432,9 +460,9 @@ public Schema AddTagField(FieldName name, bool sortable = false, bool unf = fals /// The object. public Schema AddTagField(string name, bool sortable = false, bool unf = false, bool noIndex = false, string separator = ",", - bool caseSensitive = false, bool withSuffixTrie = false) + bool caseSensitive = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false) { - Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie)); + Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie, missingIndex, emptyIndex)); return this; } @@ -445,9 +473,9 @@ public Schema AddTagField(string name, bool sortable = false, bool unf = false, /// The vector similarity algorithm to use. /// The algorithm attributes for the creation of the vector index. /// The object. - public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary? attributes = null) + public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary? attributes = null, bool missingIndex = false) { - Fields.Add(new VectorField(name, algorithm, attributes)); + Fields.Add(new VectorField(name, algorithm, attributes, missingIndex)); return this; } @@ -458,9 +486,9 @@ public Schema AddVectorField(FieldName name, VectorAlgo algorithm, DictionaryThe vector similarity algorithm to use. /// The algorithm attributes for the creation of the vector index. /// The object. - public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null) + public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null, bool missingIndex = false) { - Fields.Add(new VectorField(name, algorithm, attributes)); + Fields.Add(new VectorField(name, algorithm, attributes, missingIndex)); return this; } } diff --git a/tests/NRedisStack.Tests/Search/IndexCreationTests.cs b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs new file mode 100644 index 00000000..5ff32c32 --- /dev/null +++ b/tests/NRedisStack.Tests/Search/IndexCreationTests.cs @@ -0,0 +1,126 @@ +using StackExchange.Redis; +using NRedisStack.Search; +using NRedisStack.RedisStackCommands; +using Xunit; +using NRedisStack.Search.Literals; +using NetTopologySuite.Geometries; + +namespace NRedisStack.Tests.Search; + +public class IndexCreationTests : AbstractNRedisStackTest, IDisposable +{ + private readonly string index = "MISSING_EMPTY_INDEX"; + private static readonly string INDEXMISSING = "INDEXMISSING"; + private static readonly string INDEXEMPTY = "INDEXEMPTY"; + + public IndexCreationTests(RedisFixture redisFixture) : base(redisFixture) { } + + [Fact] + public void TestMissingEmptyFieldCommandArgs() + { + Schema sc = new Schema() + .AddTextField("text1", 1.0, missingIndex: true, emptyIndex: true) + .AddTagField("tag1", missingIndex: true, emptyIndex: true) + .AddNumericField("numeric1", missingIndex: true) + .AddGeoField("geo1", missingIndex: true) + .AddGeoShapeField("geoshape1", Schema.GeoShapeField.CoordinateSystem.FLAT, missingIndex: true) + .AddVectorField("vector1", Schema.VectorField.VectorAlgo.FLAT, missingIndex: true); + + var ftCreateParams = FTCreateParams.CreateParams(); + + var cmd = SearchCommandBuilder.Create(index, ftCreateParams, sc); + var expectedArgs = new object[] { "MISSING_EMPTY_INDEX", "SCHEMA", + "text1","TEXT",INDEXMISSING,INDEXEMPTY, + "tag1","TAG", INDEXMISSING,INDEXEMPTY, + "numeric1","NUMERIC", INDEXMISSING, + "geo1","GEO", INDEXMISSING, + "geoshape1","GEOSHAPE", "FLAT", INDEXMISSING, + "vector1","VECTOR","FLAT", INDEXMISSING}; + Assert.Equal(expectedArgs, cmd.Args); + } + + [SkipIfRedis(Is.OSSCluster, Comparison.LessThan, "7.3.240")] + public void TestMissingFields() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(2); + var vectorAttrs = new Dictionary() + { + ["TYPE"] = "FLOAT32", + ["DIM"] = "2", + ["DISTANCE_METRIC"] = "L2", + }; + Schema sc = new Schema() + .AddTextField("text1", 1.0, missingIndex: true) + .AddTagField("tag1", missingIndex: true) + .AddNumericField("numeric1", missingIndex: true) + .AddGeoField("geo1", missingIndex: true) + .AddGeoShapeField("geoshape1", Schema.GeoShapeField.CoordinateSystem.FLAT, missingIndex: true) + .AddVectorField("vector1", Schema.VectorField.VectorAlgo.FLAT, vectorAttrs, missingIndex: true); + + var ftCreateParams = FTCreateParams.CreateParams(); + Assert.True(ft.Create(index, ftCreateParams, sc)); + + var hashWithMissingFields = new HashEntry[] { new("field1", "value1"), new("field2", "value2") }; + db.HashSet("hashWithMissingFields", hashWithMissingFields); + + Polygon polygon = new GeometryFactory().CreatePolygon(new Coordinate[] { new Coordinate(1, 1), new Coordinate(10, 10), new Coordinate(100, 100), new Coordinate(1, 1), }); + + var hashWithAllFields = new HashEntry[] { new("text1", "value1"), new("tag1", "value2"), new("numeric1", "3.141"), new("geo1", "-0.441,51.458"), new("geoshape1", polygon.ToString()), new("vector1", "aaaaaaaa") }; + db.HashSet("hashWithAllFields", hashWithAllFields); + + var result = ft.Search(index, new Query("ismissing(@text1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("ismissing(@tag1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("ismissing(@numeric1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("ismissing(@geo1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("ismissing(@geoshape1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("ismissing(@vector1)")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithMissingFields", result.Documents[0].Id); + } + + [SkipIfRedis(Is.OSSCluster, Comparison.LessThan, "7.3.240")] + public void TestEmptyFields() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(2); + Schema sc = new Schema() + .AddTextField("text1", 1.0, emptyIndex: true) + .AddTagField("tag1", emptyIndex: true); + + var ftCreateParams = FTCreateParams.CreateParams(); + Assert.True(ft.Create(index, ftCreateParams, sc)); + + var hashWithMissingFields = new HashEntry[] { new("text1", ""), new("tag1", "") }; + db.HashSet("hashWithEmptyFields", hashWithMissingFields); + + var hashWithAllFields = new HashEntry[] { new("text1", "value1"), new("tag1", "value2") }; + db.HashSet("hashWithAllFields", hashWithAllFields); + + var result = ft.Search(index, new Query("@text1:''")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithEmptyFields", result.Documents[0].Id); + + result = ft.Search(index, new Query("@tag1:{''}")); + Assert.Equal(1, result.TotalResults); + Assert.Equal("hashWithEmptyFields", result.Documents[0].Id); + + } +}