Revert "documents baseline comes with CRF"

This reverts commit 7ed792a.

Author: allanj

README.md

@@ -8,10 +8,10 @@ and [benchmark results](/docs/transformers_benchmark.md) with fine-tuning BERT).
 
 | Model| Dataset | Precision | Recall | F1 |
 |-------| ------- | :---------: | :------: | :--: |
-|BERT-base-cased + CRF (this repo)| CONLL-2003 | 91.69 | 92.05 | 91.87 |
-|Roberta-base + CRF  (this repo)| CoNLL-2003 | **91.88**  | **93.01** |**92.44**|
-|BERT-base-cased + CRF  (this repo)| OntoNotes 5 |89.57  | 89.45 | 89.51 |
-|Roberta-base + CRF  (this repo)| OntoNotes 5 | **90.12**  | **91.25** |**90.68**|
+|BERT-base-cased (this repo)| CONLL-2003 | 91.69 | 92.05 | 91.87 |
+|Roberta-base (this repo)| CoNLL-2003 | **91.88**  | **93.01** |**92.44**|
+|BERT-base-cased (this repo)| OntoNotes 5 |89.57  | 89.45 | 89.51 |
+|Roberta-base (this repo)| OntoNotes 5 | **90.12**  | **91.25** |**90.68**|
 
 More [details](/docs/transformers_benchmark.md)
 

src/common/__init__.py renamed to common/__init__.py

@@ -0,0 +1,50 @@
+# 
+# @author: Allan
+#
+
+from tqdm import tqdm
+from common import Sentence, Instance
+from typing import List
+import re
+
+
+class Reader:
+
+    def __init__(self, digit2zero:bool=True):
+        """
+        Read the dataset into Instance
+        :param digit2zero: convert the digits into 0, which is a common practice for LSTM-CRF.
+        """
+        self.digit2zero = digit2zero
+        self.vocab = set()
+
+    def read_txt(self, file: str, number: int = -1) -> List[Instance]:
+        print("Reading file: " + file)
+        insts = []
+        with open(file, 'r', encoding='utf-8') as f:
+            words = []
+            ori_words = []
+            labels = []
+            for line in tqdm(f.readlines()):
+                line = line.rstrip()
+                if line == "":
+                    insts.append(Instance(Sentence(words, ori_words), labels))
+                    words = []
+                    ori_words = []
+                    labels = []
+                    if len(insts) == number:
+                        break
+                    continue
+                ls = line.split()
+                word, label = ls[0],ls[-1]
+                ori_words.append(word)
+                if self.digit2zero:
+                    word = re.sub('\d', '0', word) # replace digit with 0.
+                words.append(word)
+                self.vocab.add(word)
+                labels.append(label)
+        print("number of sentences: {}".format(len(insts)))
+        return insts
+
+
+