Merge pull request #664 from facebook/nekto/utf8_handling

Better handling of invalid UTF8 strings.

Author: ExtremeMan

Common/TaskUtil.h

@@ -22,6 +22,7 @@ typedef void (^FdOutputLineFeedBlock)(int fd, NSString *);
 typedef void (^BlockToRunWhileReading)(void);
 
 NSString *StripAnsi(NSString *inputString);
+NSString *StringFromDispatchDataWithBrokenUTF8Encoding(const char *dataPtr, size_t dataSz);
 
 /**
  *  Returns array of NSString's with contents read from fildes.

Common/TaskUtil.m

@@ -68,23 +68,41 @@
   return outputString;
 }
 
-static NSString *StringFromDispatchDataWithBrokenUTF8Encoding(const char *dataPtr, size_t dataSz)
+NSString *StringFromDispatchDataWithBrokenUTF8Encoding(const char *dataPtr, size_t dataSz)
 {
   int one = 1;
   iconv_t cd = iconv_open("UTF-8", "UTF-8");
   iconvctl(cd, ICONV_SET_DISCARD_ILSEQ, &one);
-  size_t inbytesleft = dataSz;
-  size_t outbytesleft = dataSz;
   char *inbuf  = (char *)dataPtr;
   char *outbuf = malloc(sizeof(char) * dataSz);
-  char *outptr = outbuf;
-  NSString *string = nil;
-  if (iconv(cd, &inbuf, &inbytesleft, &outptr, &outbytesleft) != (size_t)-1) {
-    string = [[NSString alloc] initWithBytes:outbuf length:dataSz - outbytesleft encoding:NSUTF8StringEncoding];
+  NSMutableString *outputString = [NSMutableString string];
+  long bytesToProcess = dataSz;
+  while (bytesToProcess > 0) {
+    NSString *string = nil;
+    size_t inbytesleft = bytesToProcess;
+    size_t outbytesleft = bytesToProcess;
+    char *outptr = outbuf;
+    size_t iconvResult = iconv(cd, &inbuf, &inbytesleft, &outptr, &outbytesleft);
+    size_t outbytesLength = bytesToProcess - outbytesleft;
+    if (outbytesLength > 0) {
+      string = [[NSString alloc] initWithBytesNoCopy:outbuf length:outbytesLength encoding:NSUTF8StringEncoding freeWhenDone:NO];
+      [outputString appendString:string];
+    }
+    if (iconvResult != (size_t)-1) {
+      inbuf += (bytesToProcess - inbytesleft);
+    } else if (errno == EINVAL) {
+      // skip first byte and then all next 10xxxxxx bytes (see UTF-8 description for more details)
+      do {
+        inbuf++;
+        inbytesleft--;
+      } while (((*inbuf) & 0xC0) == 0x80 && inbytesleft > 0);
+      [outputString appendString:@"\uFFFD"];
+    }
+    bytesToProcess = inbytesleft;
   }
   free(outbuf);
   iconv_close(cd);
-  return string;
+  return outputString;
 }
 
 static NSArray *LinesFromDispatchData(dispatch_data_t data, BOOL omitNewlineCharacters, BOOL forceUntilTheEnd, size_t *convertedSize)

xctool/xctool-tests/TaskUtilTests.m

@@ -80,4 +80,16 @@ - (void)testLaunchTaskAndFeedOutputLinesToBlockMultibyteUtf8
   }
 }
 
+- (void)testConversionToUT8OfBrokenUTF8SequenceOfBytes
+{
+  NSData *data = [NSData dataWithContentsOfFile:TEST_DATA @"BrokenUTF8EncodingInFile.txt"];
+  NSString *string = StringFromDispatchDataWithBrokenUTF8Encoding(data.bytes, data.length);
+  NSString *fixedString = [NSString stringWithContentsOfFile:TEST_DATA @"BrokenUTF8EncodingInFile-FIXED.txt" encoding:NSUTF8StringEncoding error:nil];
+  XCTAssertEqualObjects(string, fixedString);
+
+  NSString *regularString = @"qwertyuiopasdfghjk';123^&*()_<>?";
+  NSData *regularStringData = [regularString dataUsingEncoding:NSUTF8StringEncoding];
+  XCTAssertEqualObjects(StringFromDispatchDataWithBrokenUTF8Encoding(regularStringData.bytes, regularStringData.length), regularString);
+}
+
 @end