Merge pull request #133 from andrewm4894/add-anomaly-agent

Add anomaly agent

Author: andrewm4894

.example.env

@@ -73,8 +73,15 @@ ANOMSTACK_DAGSTER_SQLITE_STORAGE_BASE_DIR=tmp
 
 # OpenAI env vars for LLM based alerts
 ANOMSTACK_OPENAI_KEY=
+OPENAI_API_KEY=
 ANOMSTACK_OPENAI_MODEL=gpt-4o-mini
 
+# langsmith related env vars
+LANGSMITH_TRACING=true
+LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
+LANGSMITH_API_KEY=""
+LANGSMITH_PROJECT="anomaly-agent"
+
 # Anthropic env vars for LLM based alerts
 ANOMSTACK_ANTHROPIC_KEY=
 ANOMSTACK_ANTHROPIC_MODEL=claude-3-haiku-20240307

README.md

@@ -36,7 +36,7 @@ Painless open source anomaly detection for your metrics! 📈📉🚀
 - [Visualization](#visualization)
 - [Concepts](#concepts)
 - [Alerts](#alerts)
-- [LLM Alerts](#llm-alerts)
+- [LLM Agent Alerts](#llm-agent-alerts)
 - [Contributing](#contributing)
 
 Supported sources and databases for your metrics to live in and be queried from:
@@ -172,7 +172,7 @@ Here is a list of features of Anomstack (emoji alert warning!)
 5. 🛠️ - Ability to define your own custom python preprocess function instead of the default at [`/metrics/defaults/python/preprocess.py`](./metrics/defaults/python/preprocess.py).
 6. 📧 - Email [alerting](#alerts) with fancy(ish) ascii art plots of your metrics and anomaly scores.
 7. 💬 - Slack alerts too (want to make these nicer).
-8. 🤖 - LLM based alerts ([OpenAI](./anomstack/llm/openai.py) & [Anthropic](./anomstack/llm/anthropic.py)) - see [LLM Alerts](#llm-alerts). p.s. they don't work great yet - experimental :)
+8. 🤖 - aGeNtIc LLM based alerts - use an [anomaly-agent](https://github.com/andrewm4894/anomaly-agent) to do anomaly detection and alerting - see [LLM Agent Alerts](#llm-agent-alerts).
 9. 🕒 - Ability to ingest at whatever frequency you want and then agg to a different level for training/scoring, see [`freq`](/metrics/examples/freq/README.md) example.
 10. 📊 - Plot jobs so you can just eyeball your metrics in Dagster job logs, see [#dagster-ui-plots](#dagster-ui-plots).
 11. 🏗️ - Minimal infrastructure requirements, Anomstack just reads from and writes to whatever database you use.
@@ -557,21 +557,18 @@ Below is an example of an alert via email. Attached is a png plot with more deta
 
 ![plot](./docs/img/random_1.png)
 
-## LLM Alerts
+## LLM Agent Alerts
 
 [back to top](#anomstack)
 
-Yes! I have managed to find a way to ram a large language model (LLM) into this project. But you know what, it might just work...
+Yes! I have managed to find a way to ram a large language (LLM) ~~model~~ 🚀AGENT🚀 into this project.
 
-~~**Update**: It works horribly, but it works! 🤣. Still need to do a lot more prompt engineering to get this to work well, but it's a start.~~
-
-**Update Update**: I know how to make this work much better and more reliable + latest models are better - going to refactor this soon (done [here](https://github.com/andrewm4894/anomstack/pull/127)).
-
-Idea here is to just send the metric data and prompt to a LLM (ChatGPT) and ask it if it thinks the metric looks anomalous (and provide back an explanation). If it does, we alert.
+Idea here is to just send the metric data and prompt to an LLM Agent (built with the [anomaly-agent](https://github.com/andrewm4894/anomaly-agent)) and ask it if it thinks the metric looks anomalous (and run a verification chain to check it and also provide back an explanation for each anomaly). If it does, we alert.
 
 Notes:
 - If you don't want to send your metric data to OpenAI then just set `disable_llmalert` to `True` in your metric batch config.
-- Support for Anthropic models added [here](https://github.com/andrewm4894/anomstack/pull/128)
+- Going to add and validate [Ollama](https://ollama.com/) support soon for local LLM Agent.
+- Will be developing this more in its own library (for use in both Anomstack and any other projects) over at [anomaly-agent](https://github.com/andrewm4894/anomaly-agent) dig in there for more details.
 
 <details>
 <summary>Click to see some LLM Alert screenshots</summary>

anomstack/jobs/llmalert.py

@@ -21,7 +21,7 @@
 from anomstack.df.wrangle import wrangle_df
 from anomstack.fn.run import define_fn
 from anomstack.jinja.render import render
-from anomstack.llm.detect import detect_anomalies
+from anomstack.llm.agent import detect_anomalies
 from anomstack.sql.read import read_sql
 from anomstack.validate.validate import validate_df
 
@@ -135,20 +135,22 @@ def llmalert(context, df: pd.DataFrame) -> pd.DataFrame:
                 if llmalert_metric_rounding >= 0:
                     df_prompt = df_prompt.round(llmalert_metric_rounding)
 
-                # logger.debug(f"df_prompt: \n{df_prompt}")
-
-                prompt = make_prompt(df_prompt)
-
-                # logger.debug(f"prompt: \n{prompt}")
-
-                detected_anomalies = detect_anomalies(prompt)
-                df_detected_anomalies = pd.DataFrame(detected_anomalies)
+                df_detected_anomalies = detect_anomalies(df_prompt)
+                df_detected_anomalies = df_detected_anomalies.rename(
+                    columns={
+                        "metric_timestamp": "anomaly_timestamp",
+                        "anomaly_description": "anomaly_explanation"
+                        }
+                    )
 
                 num_anomalies_total = len(df_detected_anomalies)
                 logger.debug(f"{num_anomalies_total} total anomalies detected in {metric_name}")
 
                 # ensure both columns are datetime64[ns] type before merging
                 df_metric["metric_timestamp"] = df_metric["metric_timestamp"].dt.strftime("%Y-%m-%d %H:%M:%S")
+                df_detected_anomalies["anomaly_timestamp"] = df_detected_anomalies["anomaly_timestamp"].dt.strftime("%Y-%m-%d %H:%M:%S")
+
+                # merge the two dataframes on the metric_timestamp column
                 df_metric = df_metric.merge(
                     df_detected_anomalies,
                     how="left",

anomstack/llm/agent.py

@@ -0,0 +1,20 @@
+import pandas as pd
+
+from anomaly_agent import AnomalyAgent
+
+
+def detect_anomalies(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Detect anomalies using the AnomalyAgent.
+
+    Args:
+        df (pd.DataFrame): The input DataFrame containing metric data.
+
+    Returns:
+        pd.DataFrame: A DataFrame containing the detected anomalies.
+    """
+    anomaly_agent = AnomalyAgent()
+    anomalies = anomaly_agent.detect_anomalies(df, timestamp_col="metric_timestamp")
+    df_anomalies = anomaly_agent.get_anomalies_df(anomalies)
+
+    return df_anomalies

anomstack/llm/anthropic.py

@@ -1,4 +1,4 @@
-anthropic
+anomaly-agent
 boto3
 dagit
 dagster
@@ -17,7 +17,6 @@ libsql-experimental
 matplotlib
 numpy
 oscrypto
-openai
 pandas
 pandas-gbq
 plotly