Add delete and backup tests. Add simple parser for show line.

Author: stalkerg

tests/__init__.py

@@ -1,12 +1,14 @@
 import unittest
 
-from . import init_test, option_test, show_test
+from . import init_test, option_test, show_test, backup_test, delete_test
 
 
 def load_tests(loader, tests, pattern):
 	suite = unittest.TestSuite()
 	suite.addTests(loader.loadTestsFromModule(init_test))
 	suite.addTests(loader.loadTestsFromModule(option_test))
 	suite.addTests(loader.loadTestsFromModule(show_test))
+	suite.addTests(loader.loadTestsFromModule(backup_test))
+	suite.addTests(loader.loadTestsFromModule(delete_test))
 
 	return suite

tests/backup_test.py

@@ -0,0 +1,137 @@
+import unittest
+from os import path
+import six
+from .pb_lib import ProbackupTest
+from testgres import stop_all
+
+
+class BackupTest(ProbackupTest, unittest.TestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(BackupTest, self).__init__(*args, **kwargs)
+
+	@classmethod
+	def tearDownClass(cls):
+		stop_all()
+
+	def test_backup_modes_1(self):
+		"""standart backup modes"""
+		node = self.make_bnode('backup_modes_', base_dir="tmp_dirs/backup/backup_modes_1")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		# detect ptrack
+		is_ptrack = node.execute("postgres", "SELECT proname FROM pg_proc WHERE proname='pg_ptrack_clear'")
+		if len(is_ptrack):
+			node.append_conf("postgresql.conf", "ptrack_enable = on")
+			node.restart()
+
+		# full backup mode
+		with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		show_backup = self.show_pb(node)[0]
+		self.assertEqual(show_backup.status, six.b("OK"))
+		self.assertEqual(show_backup.mode, six.b("FULL"))
+
+		# page backup mode
+		with open(path.join(node.logs_dir, "backup_page.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"]))
+
+		show_backup = self.show_pb(node)[0]
+		self.assertEqual(show_backup.status, six.b("OK"))
+		self.assertEqual(show_backup.mode, six.b("PAGE"))
+
+		# ptrack backup mode
+		if len(is_ptrack):
+			with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log:
+				backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose"]))
+
+			show_backup = self.show_pb(node)[0]
+			self.assertEqual(show_backup.status, six.b("OK"))
+			self.assertEqual(show_backup.mode, six.b("PTRACK"))
+
+		node.stop()
+
+	def test_smooth_checkpoint_2(self):
+		"""full backup with smooth checkpoint"""
+		node = self.make_bnode('smooth_checkpoint_2', base_dir="tmp_dirs/backup/smooth_checkpoint_2")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		with open(path.join(node.logs_dir, "backup.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose", "-C"]))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("OK"))
+
+		node.stop()
+
+	def test_page_backup_without_full_3(self):
+		"""page-level backup without validated full backup"""
+		node = self.make_bnode('without_full_3', base_dir="tmp_dirs/backup/without_full_3")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		with open(path.join(node.logs_dir, "backup.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"]))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("ERROR"))
+
+		node.stop()
+
+	def test_ptrack_threads_4(self):
+		"""ptrack multi thread backup mode"""
+		node = self.make_bnode(
+			'ptrack_threads_4',
+			base_dir="tmp_dirs/backup/ptrack_threads_4",
+			options={"ptrack_enable": "on"}
+		)
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="full", options=["--verbose", "-j", "4"]))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("OK"))
+
+		with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="ptrack", options=["--verbose", "-j", "4"]))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("OK"))
+
+		node.stop()
+
+	def test_ptrack_threads_stream_5(self):
+		"""ptrack multi thread backup mode and stream"""
+		node = self.make_bnode(
+			'ptrack_threads_stream_5',
+			base_dir="tmp_dirs/backup/ptrack_threads_stream_5",
+			options={
+				"ptrack_enable": "on",
+				"max_wal_senders": "5"
+			}
+		)
+		node.append_conf("pg_hba.conf", "local replication all trust")
+		node.append_conf("pg_hba.conf", "host replication all 127.0.0.1/32 trust")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		with open(path.join(node.logs_dir, "backup_full.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(
+				node,
+				backup_type="full",
+				options=["--verbose", "-j", "4", "--stream"]
+			))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("OK"))
+
+		with open(path.join(node.logs_dir, "backup_ptrack.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(
+				node,
+				backup_type="ptrack",
+				options=["--verbose", "-j", "4", "--stream"]
+			))
+
+		self.assertEqual(self.show_pb(node)[0].status, six.b("OK"))
+
+		node.stop()

tests/delete_test.py

@@ -0,0 +1,79 @@
+import unittest
+from os import path
+import six
+from .pb_lib import ProbackupTest
+from testgres import stop_all
+import subprocess
+
+
+class DeleteTest(ProbackupTest, unittest.TestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(DeleteTest, self).__init__(*args, **kwargs)
+
+	@classmethod
+	def tearDownClass(cls):
+		stop_all()
+
+	def test_delete_full_backups_1(self):
+		"""delete full backups"""
+		node = self.make_bnode('delete_full_backups_1', base_dir="tmp_dirs/delete/delete_full_backups_1")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+		node.pgbench_init()
+
+		# full backup mode
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		pgbench = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+		pgbench.wait()
+		pgbench.stdout.close()
+
+		with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		show_backups = self.show_pb(node)
+		id_1 = show_backups[0].id
+		id_2 = show_backups[2].id
+		self.delete_pb(node, show_backups[1].id)
+		show_backups = self.show_pb(node)
+		self.assertEqual(show_backups[0].id, id_1)
+		self.assertEqual(show_backups[1].id, id_2)
+
+		node.stop()
+
+	def test_delete_increment_2(self):
+		"""delete increment and all after him"""
+		node = self.make_bnode('delete_increment_2', base_dir="tmp_dirs/delete/delete_increment_2")
+		node.start()
+		self.assertEqual(self.init_pb(node), six.b(""))
+
+		# full backup mode
+		with open(path.join(node.logs_dir, "backup_1.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, options=["--verbose"]))
+
+		# page backup mode
+		with open(path.join(node.logs_dir, "backup_2.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"]))
+
+		# page backup mode
+		with open(path.join(node.logs_dir, "backup_3.log"), "wb") as backup_log:
+			backup_log.write(self.backup_pb(node, backup_type="page", options=["--verbose"]))
+
+		show_backups = self.show_pb(node)
+		self.assertEqual(len(show_backups), 3)
+		self.delete_pb(node, show_backups[1].id)
+		show_backups = self.show_pb(node)
+		self.assertEqual(len(show_backups), 1)
+		self.assertEqual(show_backups[0].mode, six.b("FULL"))
+		self.assertEqual(show_backups[0].status, six.b("OK"))
+
+		node.stop()

tests/pb_lib.py

@@ -17,6 +17,21 @@ def dir_files(base_dir):
 	return out_list
 
 
+class ShowBackup(object):
+	def __init__(self, split_line):
+		self.id = split_line[0]
+		# TODO: parse to datetime
+		self.recovery_time = "%s %s" % (split_line[1], split_line[2])
+		self.mode = split_line[3]
+		self.cur_tli = split_line[4]
+		self.parent_tli = split_line[6]
+		# TODO: parse to interval
+		self.time = split_line[7]
+		# TODO: maybe rename to size?
+		self.data = split_line[8]
+		self.status = split_line[9]
+
+
 class ProbackupTest(object):
 	def __init__(self, *args, **kwargs):
 		super(ProbackupTest, self).__init__(*args, **kwargs)
@@ -36,13 +51,10 @@ def arcwal_dir(self, node):
 	def backup_dir(self, node):
 		return os.path.abspath("%s/backup" % node.base_dir)
 
-	def make_bnode(self, name, base_dir=None):
-		node = get_new_node('test', base_dir=path.join(self.dir_path, base_dir))
-		try:
-			node.cleanup()
-		except:
-			pass
-		shutil.rmtree(self.backup_dir(node), ignore_errors=True)
+	def make_bnode(self, name, base_dir=None, options={}):
+		real_base_dir = path.join(self.dir_path, base_dir)
+		shutil.rmtree(real_base_dir, ignore_errors=True)
+		node = get_new_node('test', base_dir=real_base_dir)
 		node.init()
 
 		node.append_conf("postgresql.conf", "wal_level = hot_standby")
@@ -51,6 +63,10 @@ def make_bnode(self, name, base_dir=None):
 			"postgresql.conf",
 			"""archive_command = 'cp "%%p" "%s/%%f"'""" % os.path.abspath(self.arcwal_dir(node))
 		)
+
+		for key, value in six.iteritems(options):
+			node.append_conf("postgresql.conf", "%s = %s" % (key, value))
+
 		return node
 
 	def run_pb(self, command):
@@ -86,7 +102,7 @@ def backup_pb(self, node, backup_type="full", options=[]):
 		# print(cmd_list)
 		return self.run_pb(cmd_list + options)
 
-	def show_pb(self, node, id=None, options=[]):
+	def show_pb(self, node, id=None, options=[], as_text=False):
 		cmd_list = [
 			"-B", self.backup_dir(node),
 			"show",
@@ -95,9 +111,12 @@ def show_pb(self, node, id=None, options=[]):
 			cmd_list += [id]
 
 		# print(cmd_list)
-		return self.run_pb(options + cmd_list)
+		if as_text:
+			return self.run_pb(options + cmd_list)
+		else:
+			return [ShowBackup(line.split()) for line in self.run_pb(options + cmd_list).splitlines()[3:]]
 
-	def validate_pb(self, node, id=None, options=[]):
+	def validate_pb(self, node, id, options=[]):
 		cmd_list = [
 			"-B", self.backup_dir(node),
 			"validate",
@@ -107,3 +126,14 @@ def validate_pb(self, node, id=None, options=[]):
 
 		# print(cmd_list)
 		return self.run_pb(options + cmd_list)
+
+	def delete_pb(self, node, id=None, options=[]):
+		cmd_list = [
+			"-B", self.backup_dir(node),
+			"delete",
+		]
+		if id:
+			cmd_list += [id]
+
+		# print(cmd_list)
+		return self.run_pb(options + cmd_list)

tests/show_test.py

@@ -25,7 +25,7 @@ def test_ok_1(self):
 			self.backup_pb(node, options=["--quiet"]),
 			six.b("")
 		)
-		self.assertIn(six.b("OK"), self.show_pb(node))
+		self.assertIn(six.b("OK"), self.show_pb(node, as_text=True))
 
 		node.stop()
 
@@ -40,10 +40,10 @@ def test_corrupt_2(self):
 			six.b("")
 		)
 
-		id_backup = self.show_pb(node).splitlines()[3].split()[0]
+		id_backup = self.show_pb(node)[0].id
 		os.remove(path.join(self.backup_dir(node), "backups", id_backup.decode("utf-8"), "database", "postgresql.conf"))
 
 		self.validate_pb(node, id_backup)
-		self.assertIn(six.b("CORRUPT"), self.show_pb(node))
+		self.assertIn(six.b("CORRUPT"), self.show_pb(node, as_text=True))
 
 		node.stop()