Skip to content

Commit 0ff0ec4

Browse files
adchiaadchia
committed
fix: Fix incorrect on demand feature view diffing and improve Java tests (#3074)
* fix: Fix ODFV bug Signed-off-by: Danny Chiao <[email protected]>
1 parent a49f70c commit 0ff0ec4

File tree

10 files changed

+344
-9
lines changed

10 files changed

+344
-9
lines changed

‎.github/workflows/java_master_only.yml

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,52 @@ jobs:
6969
java-version: '11'
7070
java-package: jdk
7171
architecture: x64
72+
- name: Setup Python (to call feast apply)
73+
uses: actions/setup-python@v2
74+
id: setup-python
75+
with:
76+
python-version: 3.8
77+
architecture: x64
78+
- name: Setup Go
79+
id: setup-go
80+
uses: actions/setup-go@v2
81+
with:
82+
go-version: 1.18.0
83+
- name: Upgrade pip version
84+
run: |
85+
pip install --upgrade "pip>=21.3.1,<22.1"
86+
- name: Get pip cache dir
87+
id: pip-cache
88+
run: |
89+
echo "::set-output name=dir::$(pip cache dir)"
90+
- name: pip cache
91+
uses: actions/cache@v2
92+
with:
93+
path: |
94+
${{ steps.pip-cache.outputs.dir }}
95+
/opt/hostedtoolcache/Python
96+
/Users/runner/hostedtoolcache/Python
97+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
98+
restore-keys: |
99+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
100+
- name: Install pip-tools
101+
run: pip install pip-tools
102+
- name: Install apache-arrow on ubuntu
103+
run: |
104+
sudo apt update
105+
sudo apt install -y -V ca-certificates lsb-release wget
106+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
107+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
108+
sudo apt update
109+
sudo apt install -y -V libarrow-dev
110+
- name: Install Python dependencies
111+
run: make install-python-ci-dependencies
112+
- uses: actions/cache@v2
113+
with:
114+
path: ~/.m2/repository
115+
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
116+
restore-keys: |
117+
${{ runner.os }}-it-maven-
72118
- uses: actions/cache@v2
73119
with:
74120
path: ~/.m2/repository
@@ -95,10 +141,46 @@ jobs:
95141
java-version: '11'
96142
java-package: jdk
97143
architecture: x64
98-
- uses: actions/setup-python@v2
144+
- name: Setup Python (to call feast apply)
145+
uses: actions/setup-python@v2
146+
id: setup-python
147+
with:
148+
python-version: 3.8
149+
architecture: x64
150+
- name: Setup Go
151+
id: setup-go
152+
uses: actions/setup-go@v2
99153
with:
100-
python-version: '3.8'
101-
architecture: 'x64'
154+
go-version: 1.18.0
155+
- name: Upgrade pip version
156+
run: |
157+
pip install --upgrade "pip>=21.3.1,<22.1"
158+
- name: Get pip cache dir
159+
id: pip-cache
160+
run: |
161+
echo "::set-output name=dir::$(pip cache dir)"
162+
- name: pip cache
163+
uses: actions/cache@v2
164+
with:
165+
path: |
166+
${{ steps.pip-cache.outputs.dir }}
167+
/opt/hostedtoolcache/Python
168+
/Users/runner/hostedtoolcache/Python
169+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
170+
restore-keys: |
171+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
172+
- name: Install pip-tools
173+
run: pip install pip-tools
174+
- name: Install apache-arrow on ubuntu
175+
run: |
176+
sudo apt update
177+
sudo apt install -y -V ca-certificates lsb-release wget
178+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
179+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
180+
sudo apt update
181+
sudo apt install -y -V libarrow-dev
182+
- name: Install Python dependencies
183+
run: make install-python-ci-dependencies
102184
- uses: actions/cache@v2
103185
with:
104186
path: ~/.m2/repository

‎.github/workflows/java_pr.yml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,52 @@ jobs:
4040
java-version: '11'
4141
java-package: jdk
4242
architecture: x64
43+
- name: Setup Python (to call feast apply)
44+
uses: actions/setup-python@v2
45+
id: setup-python
46+
with:
47+
python-version: 3.8
48+
architecture: x64
49+
- name: Setup Go
50+
id: setup-go
51+
uses: actions/setup-go@v2
52+
with:
53+
go-version: 1.18.0
54+
- name: Upgrade pip version
55+
run: |
56+
pip install --upgrade "pip>=21.3.1,<22.1"
57+
- name: Get pip cache dir
58+
id: pip-cache
59+
run: |
60+
echo "::set-output name=dir::$(pip cache dir)"
61+
- name: pip cache
62+
uses: actions/cache@v2
63+
with:
64+
path: |
65+
${{ steps.pip-cache.outputs.dir }}
66+
/opt/hostedtoolcache/Python
67+
/Users/runner/hostedtoolcache/Python
68+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
69+
restore-keys: |
70+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
71+
- name: Install pip-tools
72+
run: pip install pip-tools
73+
- name: Install apache-arrow on ubuntu
74+
run: |
75+
sudo apt update
76+
sudo apt install -y -V ca-certificates lsb-release wget
77+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
78+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
79+
sudo apt update
80+
sudo apt install -y -V libarrow-dev
81+
- name: Install Python dependencies
82+
run: make install-python-ci-dependencies
83+
- uses: actions/cache@v2
84+
with:
85+
path: ~/.m2/repository
86+
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
87+
restore-keys: |
88+
${{ runner.os }}-it-maven-
4389
- uses: actions/cache@v2
4490
with:
4591
path: ~/.m2/repository
@@ -129,6 +175,46 @@ jobs:
129175
aws-region: us-west-2
130176
- name: Use AWS CLI
131177
run: aws sts get-caller-identity
178+
- name: Setup Python (to call feast apply)
179+
uses: actions/setup-python@v2
180+
id: setup-python
181+
with:
182+
python-version: 3.8
183+
architecture: x64
184+
- name: Setup Go
185+
id: setup-go
186+
uses: actions/setup-go@v2
187+
with:
188+
go-version: 1.18.0
189+
- name: Upgrade pip version
190+
run: |
191+
pip install --upgrade "pip>=21.3.1,<22.1"
192+
- name: Get pip cache dir
193+
id: pip-cache
194+
run: |
195+
echo "::set-output name=dir::$(pip cache dir)"
196+
- name: pip cache
197+
uses: actions/cache@v2
198+
with:
199+
path: |
200+
${{ steps.pip-cache.outputs.dir }}
201+
/opt/hostedtoolcache/Python
202+
/Users/runner/hostedtoolcache/Python
203+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
204+
restore-keys: |
205+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
206+
- name: Install pip-tools
207+
run: pip install pip-tools
208+
- name: Install apache-arrow on ubuntu
209+
run: |
210+
sudo apt update
211+
sudo apt install -y -V ca-certificates lsb-release wget
212+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
213+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
214+
sudo apt update
215+
sudo apt install -y -V libarrow-dev
216+
- name: Install Python dependencies
217+
run: make install-python-ci-dependencies
132218
- name: Run integration tests
133219
run: make test-java-integration
134220
- name: Save report

‎java/CONTRIBUTING.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ mvn spotless:apply
5959
### Project Makefile
6060
The Project Makefile provides useful shorthands for common development tasks:
6161

62+
> Note: These commands rely on a local version of `feast` (Python) to be installed
6263
6364
Run all Unit tests:
6465
```

‎java/serving/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,4 +136,6 @@ Unit &amp; Integration Tests can be used to verify functionality:
136136
mvn test -pl serving --also-make
137137
# run integration tests
138138
mvn verify -pl serving --also-make
139+
# run integration tests with debugger
140+
mvn -Dmaven.failsafe.debug verify -pl serving --also-make
139141
```

‎java/serving/pom.xml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,28 @@
8282
</configuration>
8383
</plugin>
8484

85+
<!-- Call feast apply before running integration tests -->
86+
<plugin>
87+
<groupId>org.codehaus.mojo</groupId>
88+
<artifactId>exec-maven-plugin</artifactId>
89+
<version>1.6.0</version>
90+
<executions>
91+
<execution>
92+
<configuration>
93+
<executable>python</executable>
94+
<workingDirectory>src/test/resources/docker-compose/feast10/</workingDirectory>
95+
<arguments>
96+
<argument>setup_it.py</argument>
97+
</arguments>
98+
</configuration>
99+
<id>feast_test_apply</id>
100+
<phase>process-test-resources</phase>
101+
<goals>
102+
<goal>exec</goal>
103+
</goals>
104+
</execution>
105+
</executions>
106+
</plugin>
85107
</plugins>
86108
</build>
87109

‎java/serving/src/test/resources/docker-compose/feast10/definitions.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,17 +73,17 @@ def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame:
7373

7474
entity = Entity(name="entity", value_type=ValueType.STRING,)
7575

76-
benchmark_feature_views = [
77-
FeatureView(
76+
benchmark_feature_views = []
77+
for i in range(25):
78+
fv = FeatureView(
7879
name=f"feature_view_{i}",
7980
entities=[entity],
8081
ttl=Duration(seconds=86400),
8182
schema=[Field(name=f"feature_{10 * i + j}", dtype=Int64) for j in range(10)],
8283
online=True,
8384
source=generated_data_source,
8485
)
85-
for i in range(25)
86-
]
86+
benchmark_feature_views.append(fv)
8787

8888
benchmark_feature_service = FeatureService(
8989
name=f"benchmark_feature_service", features=benchmark_feature_views,

‎java/serving/src/test/resources/docker-compose/feast10/registry.db

-13.9 KB
Binary file not shown.

‎java/serving/src/test/resources/docker-compose/feast10/setup_it.py

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
from pathlib import Path
2+
from feast.repo_config import load_repo_config
3+
from datetime import datetime, timedelta
4+
5+
import numpy as np
6+
import pandas as pd
7+
8+
from definitions import (
9+
benchmark_feature_service,
10+
benchmark_feature_views,
11+
driver,
12+
driver_hourly_stats_view,
13+
entity,
14+
transformed_conv_rate,
15+
)
16+
17+
from feast import FeatureStore
18+
19+
20+
def setup_data():
21+
start = datetime.now() - timedelta(days=10)
22+
23+
df = pd.DataFrame()
24+
df["driver_id"] = np.arange(1000, 1010)
25+
df["created"] = datetime.now()
26+
df["conv_rate"] = np.arange(0, 1, 0.1)
27+
df["acc_rate"] = np.arange(0.5, 1, 0.05)
28+
df["avg_daily_trips"] = np.arange(0, 1000, 100)
29+
30+
# some of rows are beyond 7 days to test OUTSIDE_MAX_AGE status
31+
df["event_timestamp"] = start + pd.Series(np.arange(0, 10)).map(
32+
lambda days: timedelta(days=days)
33+
)
34+
35+
# Store data in parquet files. Parquet is convenient for local development mode. For
36+
# production, you can use your favorite DWH, such as BigQuery. See Feast documentation
37+
# for more info.
38+
df.to_parquet("driver_stats.parquet")
39+
40+
# For Benchmarks
41+
# Please read more in Feast RFC-031
42+
# (link https://docs.google.com/document/d/12UuvTQnTTCJhdRgy6h10zSbInNGSyEJkIxpOcgOen1I/edit)
43+
# about this benchmark setup
44+
def generate_data(
45+
num_rows: int, num_features: int, destination: str
46+
) -> pd.DataFrame:
47+
features = [f"feature_{i}" for i in range(num_features)]
48+
columns = ["entity", "event_timestamp"] + features
49+
df = pd.DataFrame(0, index=np.arange(num_rows), columns=columns)
50+
df["event_timestamp"] = datetime.utcnow()
51+
for column in features:
52+
df[column] = np.random.randint(1, num_rows, num_rows)
53+
54+
df["entity"] = "key-" + pd.Series(np.arange(1, num_rows + 1)).astype(
55+
pd.StringDtype()
56+
)
57+
58+
df.to_parquet(destination)
59+
60+
generate_data(10**3, 250, "benchmark_data.parquet")
61+
62+
63+
def main():
64+
print("Running setup_it.py")
65+
66+
setup_data()
67+
existing_repo_config = load_repo_config(Path("."))
68+
69+
# Update to default online store since otherwise, relies on Dockerized Redis service
70+
fs = FeatureStore(config=existing_repo_config.copy(update={"online_store": {}}))
71+
fs.apply(
72+
[
73+
driver_hourly_stats_view,
74+
transformed_conv_rate,
75+
driver,
76+
entity,
77+
benchmark_feature_service,
78+
*benchmark_feature_views,
79+
]
80+
)
81+
82+
print("setup_it finished")
83+
84+
85+
if __name__ == "__main__":
86+
main()

‎sdk/python/feast/diff/registry_diff.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,8 @@ def diff_registry_objects(
144144
continue
145145
elif getattr(current_spec, _field.name) != getattr(new_spec, _field.name):
146146
if _field.name == "user_defined_function":
147-
current_spec = cast(OnDemandFeatureViewSpec, current_proto)
148-
new_spec = cast(OnDemandFeatureViewSpec, new_proto)
147+
current_spec = cast(OnDemandFeatureViewSpec, current_spec)
148+
new_spec = cast(OnDemandFeatureViewSpec, new_spec)
149149
current_udf = current_spec.user_defined_function
150150
new_udf = new_spec.user_defined_function
151151
for _udf_field in current_udf.DESCRIPTOR.fields:

0 commit comments

Comments
 (0)