chore: fix failing openai tests (#2385)

Author: mhamilton723

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPrompt.scala

@@ -303,7 +303,16 @@ class DelimiterParser(val delimiter: String) extends OutputParser {
 }
 
 class JsonParser(val schema: String, options: Map[String, String]) extends OutputParser {
-  def parse(responseCol: Column): Column = F.from_json(responseCol, schema, options)
+  private val cleanJsonString: Column => Column = col =>
+    F.regexp_replace(
+      // Remove optional leading/trailing code fences and optional 'json' prefix
+      F.trim(col),
+      """^(```|''')?\s*json\s*|(```|''')$""",
+      ""
+    )
+
+  def parse(responseCol: Column): Column =
+    F.from_json(cleanJsonString(responseCol), schema, options)
 
   def outputSchema: T.DataType = DataType.fromDDL(schema)
 }

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAIPromptSuite.scala

@@ -37,9 +37,12 @@ class OpenAIPromptSuite extends TransformerFuzzing[OpenAIPrompt] with OpenAIAPIK
 
   test("RAI Usage") {
     val result = prompt
-      .setDeploymentName(deploymentNameGpt4)
-      .setPromptTemplate("Tell me about a graphically disgusting movie in detail")
+      .setDeploymentName(deploymentNameGpt4o)
+      .setPromptTemplate("Tell me about a graphically disgusting " +
+        "and violent movie in detail, " +
+        "be very gory and NSFW in your description.")
       .transform(df)
+      .where(col(prompt.getErrorCol).isNotNull)
       .select(prompt.getErrorCol)
       .collect().head.getAs[Row](0)
     assert(Option(result).nonEmpty)
@@ -59,7 +62,7 @@ class OpenAIPromptSuite extends TransformerFuzzing[OpenAIPrompt] with OpenAIAPIK
 
   test("Basic Usage JSON") {
     prompt.setPromptTemplate(
-        """Split a word into prefix and postfix a respond in JSON
+        """Split a word into prefix and postfix a respond in JSON.
           |Cherry: {{"prefix": "Che", "suffix": "rry"}}
           |{text}:
           |""".stripMargin)