Small fixes

src/archive.c

@@ -512,8 +512,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d
 		pg_crc32 crc32_src;
 		pg_crc32 crc32_dst;
 
-		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
-		crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false);
+		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
+		crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false);
 
 		if (crc32_src == crc32_dst)
 		{
@@ -760,9 +760,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir,
 		pg_crc32 crc32_src;
 		pg_crc32 crc32_dst;
 
-		/* TODO: what if one of them goes missing? */
-		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
-		crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true);
+		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
+		crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false);
 
 		if (crc32_src == crc32_dst)
 		{

src/catalog.c

@@ -1132,6 +1132,9 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
 			file->hdr_size = (int) hdr_size;
 
+		if (file->external_dir_num == 0)
+			set_forkname(file);
+
 		parray_append(files, file);
 	}
 
@@ -2488,7 +2491,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 	char		control_path[MAXPGPATH];
 	char		control_path_temp[MAXPGPATH];
 	size_t		i = 0;
-	#define BUFFERSZ 1024*1024
+	#define BUFFERSZ (1024*1024)
 	char		*buf;
 	int64 		backup_size_on_disk = 0;
 	int64 		uncompressed_size_on_disk = 0;

src/data.c

@@ -801,8 +801,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
 		(prev_file && file->exists_in_prev &&
 		 file->mtime <= parent_backup_time))
 	{
-
-		file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
+		/*
+		 * file could be deleted under our feets.
+		 * But then backup_non_data_file_internal will handle it safely
+		 */
+		file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
 
 		/* ...and checksum is the same... */
 		if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1327,7 +1330,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 	if (already_exists)
 	{
 		/* compare checksums of already existing file and backup file */
-		pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false);
+		pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
 
 		if (file_crc == tmp_file->crc)
 		{

src/dir.c

@@ -758,57 +758,22 @@ dir_check_file(pgFile *file, bool backup_logs)
 			return CHECK_FALSE;
 		else if (isdigit(file->name[0]))
 		{
-			char	   *fork_name;
-			int			len;
-			char		suffix[MAXPGPATH];
+			set_forkname(file);
 
-			fork_name = strstr(file->name, "_");
-			if (fork_name)
-			{
-				/* Auxiliary fork of the relfile */
-				if (strcmp(fork_name, "_vm") == 0)
-					file->forkName = vm;
-
-				else if (strcmp(fork_name, "_fsm") == 0)
-					file->forkName = fsm;
-
-				else if (strcmp(fork_name, "_cfm") == 0)
-					file->forkName = cfm;
-
-				else if (strcmp(fork_name, "_ptrack") == 0)
-					file->forkName = ptrack;
-
-				else if (strcmp(fork_name, "_init") == 0)
-					file->forkName = init;
-
-				// extract relOid for certain forks
-				if (file->forkName == vm ||
-					file->forkName == fsm ||
-					file->forkName == init ||
-					file->forkName == cfm)
-				{
-					// sanity
-					if (sscanf(file->name, "%u_*", &(file->relOid)) != 1)
-						file->relOid = 0;
-				}
+			if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */
+				return CHECK_FALSE;
+			else if (file->forkName != none)
+				return CHECK_TRUE;
 
-				/* Do not backup ptrack files */
-				if (file->forkName == ptrack)
-					return CHECK_FALSE;
-			}
-			else
+			/* Set is_datafile flag */
 			{
+				char suffix[MAXFNAMELEN];
 
-				len = strlen(file->name);
-				/* reloid.cfm */
-				if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0)
-					return CHECK_TRUE;
-
+				/* check if file is datafile */
 				sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
 									&(file->segno), suffix);
-				if (sscanf_res == 0)
-					elog(ERROR, "Cannot parse file name \"%s\"", file->name);
-				else if (sscanf_res == 1 || sscanf_res == 2)
+				Assert(sscanf_res > 0); /* since first char is digit */
+				if (sscanf_res == 1 || sscanf_res == 2)
 					file->is_datafile = true;
 			}
 		}
@@ -1954,3 +1919,35 @@ pfilearray_clear_locks(parray *file_list)
 		pg_atomic_clear_flag(&file->lock);
 	}
 }
+
+/* Set forkName if possible */
+void
+set_forkname(pgFile *file)
+{
+	int name_len = strlen(file->name);
+
+	/* Auxiliary fork of the relfile */
+	if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0)
+		file->forkName = vm;
+
+	else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0)
+		file->forkName = fsm;
+
+	else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0)
+		file->forkName = cfm;
+
+	else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0)
+		file->forkName = init;
+
+	else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0)
+		file->forkName = ptrack;
+
+	// extract relOid for certain forks
+
+	if ((file->forkName == vm ||
+		 file->forkName == fsm ||
+		 file->forkName == init ||
+		 file->forkName == cfm) &&
+		(sscanf(file->name, "%u*", &(file->relOid)) != 1))
+		file->relOid = 0;
+}

src/pg_probackup.c

@@ -88,7 +88,7 @@ bool perm_slot = false;
 /* backup options */
 bool         backup_logs = false;
 bool         smooth_checkpoint;
-char        *remote_agent;
+bool         remote_agent = false;
 static char *backup_note = NULL;
 /* catchup options */
 static char *catchup_source_pgdata = NULL;
@@ -361,6 +361,7 @@ main(int argc, char *argv[])
 					elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' "
 							"is launched as an agent for pg_probackup binary with version '%s'",
 							PROGRAM_VERSION, argv[2]);
+				remote_agent = true;
 				fio_communicate(STDIN_FILENO, STDOUT_FILENO);
 				return 0;
 			case HELP_CMD:

src/pg_probackup.h

@@ -215,6 +215,7 @@ typedef enum CompressAlg
 
 typedef enum ForkName
 {
+	none,
 	vm,
 	fsm,
 	cfm,
@@ -798,7 +799,7 @@ extern bool perm_slot;
 extern bool		smooth_checkpoint;
 
 /* remote probackup options */
-extern char* remote_agent;
+extern bool remote_agent;
 
 extern bool exclusive_backup;
 
@@ -1091,6 +1092,7 @@ extern int pgCompareString(const void *str1, const void *str2);
 extern int pgPrefixCompareString(const void *str1, const void *str2);
 extern int pgCompareOid(const void *f1, const void *f2);
 extern void pfilearray_clear_locks(parray *file_list);
+extern void set_forkname(pgFile *file);
 
 /* in data.c */
 extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,

src/utils/configuration.c

@@ -531,7 +531,6 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[],
 			opt = option_find(c, options);
 
 		if (opt
-			&& !remote_agent
 			&& opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT)
 			elog(ERROR, "Option %s cannot be specified in command line",
 				 opt->lname);

src/utils/file.c

@@ -1355,9 +1355,15 @@ fio_sync(char const* path, fio_location location)
 	}
 }
 
+enum {
+	GET_CRC32_DECOMPRESS = 1,
+	GET_CRC32_MISSING_OK = 2
+};
+
 /* Get crc32 of file */
 pg_crc32
-fio_get_crc32(const char *file_path, fio_location location, bool decompress)
+fio_get_crc32(const char *file_path, fio_location location,
+			  bool decompress, bool missing_ok)
 {
 	if (fio_is_remote(location))
 	{
@@ -1370,7 +1376,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
 		hdr.arg = 0;
 
 		if (decompress)
-			hdr.arg = 1;
+			hdr.arg = GET_CRC32_DECOMPRESS;
+		if (missing_ok)
+			hdr.arg |= GET_CRC32_MISSING_OK;
 
 		IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
 		IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len);
@@ -1381,9 +1389,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
 	else
 	{
 		if (decompress)
-			return pgFileGetCRCgz(file_path, true, true);
+			return pgFileGetCRCgz(file_path, true, missing_ok);
 		else
-			return pgFileGetCRC(file_path, true, true);
+			return pgFileGetCRC(file_path, true, missing_ok);
 	}
 }
 
@@ -3380,10 +3388,10 @@ fio_communicate(int in, int out)
 			break;
 		  case FIO_GET_CRC32:
 			/* calculate crc32 for a file */
-			if (hdr.arg == 1)
-				crc = pgFileGetCRCgz(buf, true, true);
+			if ((hdr.arg & GET_CRC32_DECOMPRESS))
+				crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			else
-				crc = pgFileGetCRC(buf, true, true);
+				crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
 			break;
 		  case FIO_GET_CHECKSUM_MAP:

src/utils/file.h

@@ -120,7 +120,8 @@ extern int     fio_truncate(int fd, off_t size);
 extern int     fio_close(int fd);
 extern void    fio_disconnect(void);
 extern int     fio_sync(char const* path, fio_location location);
-extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress);
+extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
+							  bool decompress, bool missing_ok);
 
 extern int     fio_rename(char const* old_path, char const* new_path, fio_location location);
 extern int     fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

tests/cfs_backup.py

@@ -995,6 +995,11 @@ def test_delete_random_cfm_file_from_tablespace_dir(self):
             "FROM generate_series(0,256) i".format('t1', tblspace_name)
         )
 
+        self.node.safe_psql(
+            "postgres",
+            "CHECKPOINT"
+        )
+
         list_cmf = find_by_extensions(
             [self.get_tblspace_path(self.node, tblspace_name)],
             ['.cfm'])
@@ -1044,6 +1049,11 @@ def test_delete_random_data_file_from_tablespace_dir(self):
             "FROM generate_series(0,256) i".format('t1', tblspace_name)
         )
 
+        self.node.safe_psql(
+            "postgres",
+            "CHECKPOINT"
+        )
+
         list_data_files = find_by_pattern(
             [self.get_tblspace_path(self.node, tblspace_name)],
             '^.*/\d+$')

tests/cfs_restore.py

@@ -103,6 +103,7 @@ def test_restore_empty_tablespace_from_fullbackup(self):
             "postgres",
             "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name)
         )
+        tblspace = str(tblspace)
         self.assertTrue(
             tblspace_name in tblspace and "compression=true" in tblspace,
             "ERROR: The tablespace not restored or it restored without compressions"