Merge pull request #134 from ntumlgroup/autotest-linear

Added linear tests and save k prediction

Author: Eleven1Liu

linear_trainer.py

@@ -1,9 +1,11 @@
 import os
+import logging
 from math import ceil
 
 import numpy as np
 
 import libmultilabel.linear as linear
+from libmultilabel.utils import dump_log, argsort_top_k
 
 
 def linear_test(config, model, datasets):
@@ -13,12 +15,24 @@ def linear_test(config, model, datasets):
         datasets['test']['y'].shape[1]
     )
     num_instance = datasets['test']['x'].shape[0]
+
+    k = config.save_k_predictions
+    top_k_idx = np.zeros((num_instance, k), dtype='i')
+    top_k_scores = np.zeros((num_instance, k), dtype='d')
+
     for i in range(ceil(num_instance / config.eval_batch_size)):
         slice = np.s_[i*config.eval_batch_size:(i+1)*config.eval_batch_size]
         preds = linear.predict_values(model, datasets['test']['x'][slice])
         target = datasets['test']['y'][slice].toarray()
         metrics.update(preds, target)
-    print(linear.tabulate_metrics(metrics.compute(), 'test'))
+
+        if k > 0:
+            top_k_idx[slice] = argsort_top_k(preds, k, axis=1)
+            top_k_scores[slice] = np.take_along_axis(
+                preds, top_k_idx[slice], axis=1)
+
+    metric_dict = metrics.compute()
+    return (metric_dict, top_k_idx, top_k_scores)
 
 
 def linear_train(datasets, config):
@@ -46,5 +60,18 @@ def linear_run(config):
         linear.save_pipeline(config.checkpoint_dir, preprocessor, model)
 
     if os.path.exists(config.test_path):
-        linear_test(config, model, datasets)
-    # TODO: dump logs?
+        metric_dict, top_k_idx, top_k_scores = linear_test(
+            config, model, datasets)
+
+        dump_log(config=config, metrics=metric_dict,
+                 split='test', log_path=config.log_path)
+        print(linear.tabulate_metrics(metric_dict, 'test'))
+
+        if config.save_k_predictions > 0:
+            classes = preprocessor.binarizer.classes_
+            with open(config.predict_out_path, 'w') as fp:
+                for idx, score in zip(top_k_idx, top_k_scores):
+                    out_str = ' '.join([f'{classes[i]}:{s:.4}' for i, s in zip(
+                        idx, score)])
+                    fp.write(out_str+'\n')
+            logging.info(f'Saved predictions to: {config.predict_out_path}')

main.py

@@ -145,6 +145,7 @@ def get_config():
     )
     config.checkpoint_dir = os.path.join(config.result_dir, config.run_name)
     config.log_path = os.path.join(config.checkpoint_dir, 'logs.json')
+    config.predict_out_path = config.predict_out_path or os.path.join(config.checkpoint_dir, 'predictions.txt')
 
     config.train_path = config.train_path or os.path.join(config.data_dir, 'train.txt')
     config.val_path = config.val_path or os.path.join(config.data_dir, 'valid.txt')

tests/autotest.sh

@@ -70,6 +70,14 @@ main() {
     run_test "rcv1" "kim_cnn" "$template"
   done
 
+  TEST_COMMAND_TEMPLATES=(
+    # Run default linear 1vsrest
+    "python3 main.py --config example_config/%s/%s.yml --result_dir $RESULT_DIR"
+  )
+  for template in "${TEST_COMMAND_TEMPLATES[@]}"; do
+    run_test "rcv1" "l2svm" "$template"
+  done
+
   # Print the test results and remove the intermediate files.
   all_tests=$(less $REPORT_PATH | wc -l)
   passed_tests=$(grep "PASSED" $REPORT_PATH | wc -l)
@@ -87,4 +95,4 @@ if $(echo $(pwd) | grep -q "tests"); then
   echo "Go to the LibMultilabel directory and run: bash tests/autotest.sh"
 else
   main
-fi
+fi

torch_trainer.py

@@ -196,11 +196,7 @@ def test(self, split='test'):
         metric_dict = self.trainer.test(self.model, test_dataloaders=test_loader)[0]
 
         if self.config.save_k_predictions > 0:
-            if not self.config.predict_out_path:
-                predict_out_path = os.path.join(self.checkpoint_dir, 'predictions.txt')
-            else:
-                predict_out_path = self.config.predict_out_path
-            self._save_predictions(test_loader, predict_out_path)
+            self._save_predictions(test_loader, self.config.predict_out_path)
 
         return metric_dict