add base concept of entities to signals (#2977)

* add base concenpt of entities to signals

* Add entity model to store type results and add entity case tab

* Add missing import and remove unused import

* Remove debug statements and sleep from find_entities function

* Fix entity_type update service and fix signal instance popover in table

* fix service tests, signal_instance id should be uuid not integer

* Remove debug console.log statement

* move find_entities function to entity service

* Update comment

* Remove global find from front-end, add regex help message about groups

* Address @kglisson comments

* Make case edit sheet wider to give tabs some room to breathe

Author: wssheldon

src/dispatch/api.py

@@ -21,6 +21,8 @@
 from dispatch.data.source.views import router as source_router
 from dispatch.definition.views import router as definition_router
 from dispatch.document.views import router as document_router
+from dispatch.entity.views import router as entity_router
+from dispatch.entity_type.views import router as entity_type_router
 from dispatch.feedback.views import router as feedback_router
 from dispatch.incident.priority.views import router as incident_priority_router
 from dispatch.incident.severity.views import router as incident_severity_router
@@ -131,6 +133,12 @@ def get_organization_path(organization: OrganizationSlug):
 authenticated_organization_api_router.include_router(
     document_router, prefix="/documents", tags=["documents"]
 )
+authenticated_organization_api_router.include_router(
+    entity_router, prefix="/entity", tags=["entities"]
+)
+authenticated_organization_api_router.include_router(
+    entity_type_router, prefix="/entity_type", tags=["entity_types"]
+)
 authenticated_organization_api_router.include_router(tag_router, prefix="/tags", tags=["tags"])
 authenticated_organization_api_router.include_router(
     tag_type_router, prefix="/tag_types", tags=["tag_types"]

src/dispatch/case/models.py

@@ -25,6 +25,7 @@
 from dispatch.database.core import Base
 from dispatch.document.models import Document, DocumentRead
 from dispatch.enums import Visibility
+from dispatch.entity.models import EntityRead
 from dispatch.event.models import EventRead
 from dispatch.group.models import Group, GroupRead
 from dispatch.incident.models import IncidentReadMinimal
@@ -165,6 +166,7 @@ class SignalRead(DispatchBase):
 
 class SignalInstanceRead(DispatchBase):
     signal: SignalRead
+    entities: Optional[List[EntityRead]] = []
     tags: Optional[List[TagRead]] = []
     raw: Any
     fingerprint: str

src/dispatch/case/service.py

@@ -195,12 +195,13 @@ def create(*, db_session, case_in: CaseCreate, current_user: DispatchUser = None
     )
 
     # add reporter
-    participant_flows.add_participant(
-        case_in.reporter.individual.email,
-        case,
-        db_session,
-        role=ParticipantRoleType.reporter,
-    )
+    if case_in.reporter:
+        participant_flows.add_participant(
+            case_in.reporter.individual.email,
+            case,
+            db_session,
+            role=ParticipantRoleType.reporter,
+        )
 
     return case
 

src/dispatch/database/revisions/tenant/versions/2023-02-09_8746b4e292d2.py

@@ -0,0 +1,102 @@
+"""Adds entity and entity type tables and associations
+
+Revision ID: 8746b4e292d2
+Revises: 941efd922446
+Create Date: 2023-02-09 23:18:11.326027
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import TSVECTOR, UUID
+
+# revision identifiers, used by Alembic.
+revision = "8746b4e292d2"
+down_revision = "941efd922446"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust!
+
+    # EntityType
+    op.create_table(
+        "entity_type",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("field", sa.String(), nullable=True),
+        sa.Column("regular_expression", sa.String(), nullable=True),
+        sa.Column("global_find", sa.Boolean(), nullable=True),
+        sa.Column("enabled", sa.Boolean(), nullable=True),
+        sa.Column("search_vector", TSVECTOR, nullable=True),
+        sa.Column("project_id", sa.Integer(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "project_id"),
+    )
+    op.create_table(
+        "assoc_signal_entity_types",
+        sa.Column("signal_id", sa.Integer(), nullable=False),
+        sa.Column("entity_type_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(["signal_id"], ["signal.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["entity_type_id"], ["entity_type.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("signal_id", "entity_type_id"),
+    )
+    op.create_index(
+        "entity_type_search_vector_idx",
+        "entity_type",
+        ["search_vector"],
+        unique=False,
+        postgresql_using="gin",
+    )
+
+    # Entity
+    op.create_table(
+        "entity",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("value", sa.String(), nullable=True),
+        sa.Column("source", sa.Boolean(), nullable=True),
+        sa.Column("entity_type_id", sa.Integer(), nullable=False),
+        sa.Column("search_vector", TSVECTOR, nullable=True),
+        sa.Column("project_id", sa.Integer(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["entity_type_id"],
+            ["entity_type.id"],
+        ),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "project_id"),
+    )
+    op.create_index(
+        "ix_entity_search_vector",
+        "entity",
+        ["search_vector"],
+        unique=False,
+        postgresql_using="gin",
+    )
+    op.create_table(
+        "assoc_signal_instance_entities",
+        sa.Column("signal_instance_id", UUID(), nullable=False),
+        sa.Column("entity_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(["signal_instance_id"], ["signal_instance.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("signal_instance_id", "entity_id"),
+    )
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_entity_search_vector", table_name="entity")
+    op.drop_table("entity")
+    op.drop_table("entity_type")
+    op.drop_index("entity_type_search_vector_idx", table_name="entity", postgresql_using="gin")
+    op.drop_table("assoc_signal_entity_types")
+    op.drop_table("assoc_signal_instance_entity_types")
+    # ### end Alembic commands ###

src/dispatch/entity/__init__.py

@@ -0,0 +1,81 @@
+from typing import Optional, List
+from pydantic import Field
+
+from sqlalchemy import Column, Integer, String, ForeignKey
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql.schema import UniqueConstraint
+from sqlalchemy_utils import TSVectorType
+
+from dispatch.database.core import Base
+from dispatch.models import DispatchBase, TimeStampMixin, ProjectMixin, PrimaryKey
+from dispatch.project.models import ProjectRead
+from dispatch.entity_type.models import (
+    EntityTypeCreate,
+    EntityTypeRead,
+    EntityTypeReadMinimal,
+    EntityTypeUpdate,
+)
+
+
+class Entity(Base, TimeStampMixin, ProjectMixin):
+    __table_args__ = (UniqueConstraint("name", "project_id"),)
+
+    # Columns
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    description = Column(String)
+    value = Column(String)
+    source = Column(String)
+
+    # Relationships
+    entity_type_id = Column(Integer, ForeignKey("entity_type.id"), nullable=False)
+    entity_type = relationship("EntityType", backref="entity")
+
+    # the catalog here is simple to help matching "named entities"
+    search_vector = Column(
+        TSVectorType(
+            "name",
+            "description",
+            weights={"name": "A", "description": "B"},
+            regconfig="pg_catalog.simple",
+        )
+    )
+
+
+# Pydantic models
+class EntityBase(DispatchBase):
+    name: Optional[str] = Field(None, nullable=True)
+    source: Optional[str] = Field(None, nullable=True)
+    value: Optional[str] = Field(None, nullable=True)
+    description: Optional[str] = Field(None, nullable=True)
+
+
+class EntityCreate(EntityBase):
+    id: Optional[PrimaryKey]
+    entity_type: EntityTypeCreate
+    project: ProjectRead
+
+
+class EntityUpdate(EntityBase):
+    id: Optional[PrimaryKey]
+    entity_type: Optional[EntityTypeUpdate]
+
+
+class EntityRead(EntityBase):
+    id: PrimaryKey
+    entity_type: Optional[EntityTypeRead]
+    project: ProjectRead
+
+
+class EntityReadMinimal(DispatchBase):
+    id: PrimaryKey
+    name: Optional[str] = Field(None, nullable=True)
+    source: Optional[str] = Field(None, nullable=True)
+    value: Optional[str] = Field(None, nullable=True)
+    description: Optional[str] = Field(None, nullable=True)
+    entity_type: Optional[EntityTypeReadMinimal]
+
+
+class EntityPagination(DispatchBase):
+    items: List[EntityRead]
+    total: int