tests: Fix numerous resource leaks

contrib/pyln-client/pyln/client/lightning.py

@@ -254,9 +254,12 @@ def connect(self) -> None:
                 # Open an fd to our home directory, that we can then find
                 # through `/proc/self/fd` and access the contents.
                 dirfd = os.open(dirname, os.O_DIRECTORY | os.O_RDONLY)
-                short_path = "/proc/self/fd/%d/%s" % (dirfd, basename)
-                self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-                self.sock.connect(short_path)
+                try:
+                    short_path = "/proc/self/fd/%d/%s" % (dirfd, basename)
+                    self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                    self.sock.connect(short_path)
+                finally:
+                    os.close(dirfd)
             elif (e.args[0] == "AF_UNIX path too long" and os.uname()[0] == "Darwin"):
                 temp_dir = tempfile.mkdtemp()
                 temp_link = os.path.join(temp_dir, "socket_link")

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -512,6 +512,9 @@ def map_node_error(nodes, f, msg):
     if not ok:
         map_node_error(nf.nodes, prinErrlog, "some node failed unexpected, non-empty errlog file")
 
+    for n in nf.nodes:
+        n.daemon.cleanup_files()
+
 
 def getErrlog(node):
     for error_file in os.listdir(node.daemon.lightning_dir):

contrib/pyln-testing/pyln/testing/gossip.py

@@ -16,7 +16,13 @@ def __init__(self, path: Path):
         self.path = path
         self.log = logging.getLogger("GossipStore")
 
+    def __del__(self):
+        if self.fd is not None:
+            self.fd.close()
+
     def open(self):
+        if self.fd is not None:
+            self.fd.close()
         self.fd = self.path.open(mode="rb")
         self.version = ord(self.fd.read(1))
         if self.version < 3:

contrib/pyln-testing/pyln/testing/utils.py

@@ -60,7 +60,8 @@ def env(name, default=None):
     """
     fname = 'config.vars'
     if os.path.exists(fname):
-        lines = open(fname, 'r').readlines()
+        with open(fname, 'r') as f:
+            lines = f.readlines()
         config = dict([(line.rstrip().split('=', 1)) for line in lines])
     else:
         config = {}
@@ -938,7 +939,7 @@ def grpc(self):
 
         import grpc
         p = Path(self.daemon.lightning_dir) / TEST_NETWORK
-        cert, key, ca = [f.open('rb').read() for f in [
+        cert, key, ca = [f.read_bytes() for f in [
             p / 'client.pem',
             p / 'client-key.pem',
             p / "ca.pem"]]
@@ -1689,16 +1690,16 @@ def get_node(self, node_id=None, options=None, dbfile=None,
         self.nodes.append(node)
         self.reserved_ports.append(port)
         if dbfile:
-            out = open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
-                                    'lightningd.sqlite3'), 'xb')
-            with lzma.open(os.path.join('tests/data', dbfile), 'rb') as f:
-                out.write(f.read())
+            with open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
+                                   'lightningd.sqlite3'), 'xb') as out:
+                with lzma.open(os.path.join('tests/data', dbfile), 'rb') as f:
+                    out.write(f.read())
 
         if bkpr_dbfile:
-            out = open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
-                                    'accounts.sqlite3'), 'xb')
-            with lzma.open(os.path.join('tests/data', bkpr_dbfile), 'rb') as f:
-                out.write(f.read())
+            with open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
+                                   'accounts.sqlite3'), 'xb') as out:
+                with lzma.open(os.path.join('tests/data', bkpr_dbfile), 'rb') as f:
+                    out.write(f.read())
 
         if gossip_store_file:
             shutil.copy(gossip_store_file, os.path.join(node.daemon.lightning_dir, TEST_NETWORK,

tests/fixtures.py

@@ -58,7 +58,8 @@ class CompatLevel(object):
     """
     def __init__(self):
         makefile = os.path.join(os.path.dirname(__file__), "..", "Makefile")
-        lines = [l for l in open(makefile, 'r') if l.startswith('COMPAT_CFLAGS')]
+        with open(makefile, 'r') as f:
+            lines = [l for l in f if l.startswith('COMPAT_CFLAGS')]
         assert(len(lines) == 1)
         line = lines[0]
         flags = re.findall(r'COMPAT_V([0-9]+)=1', line)

tests/test_bookkeeper.py

@@ -690,15 +690,15 @@ def test_bookkeeping_descriptions(node_factory, bitcoind, chainparams):
     l1.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv')
     l2.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv')
     koinly_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv')
-    l1_koinly_csv = open(koinly_path, 'rb').read()
+    l1_koinly_csv = Path(koinly_path).read_bytes()
     bolt11_exp = bytes('invoice,"test \'bolt11\' description, 🥰🪢",', 'utf-8')
     bolt12_exp = bytes('invoice,"test \'bolt12\' description, 🥰🪢",', 'utf-8')
 
     assert l1_koinly_csv.find(bolt11_exp) >= 0
     assert l1_koinly_csv.find(bolt12_exp) >= 0
 
     koinly_path = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv')
-    l2_koinly_csv = open(koinly_path, 'rb').read()
+    l2_koinly_csv = Path(koinly_path).read_bytes()
     assert l2_koinly_csv.find(bolt11_exp) >= 0
     assert l2_koinly_csv.find(bolt12_exp) >= 0
 

tests/test_cln_rs.py

@@ -129,9 +129,9 @@ def test_grpc_connect(node_factory):
     key_path = p / "client-key.pem"
     ca_cert_path = p / "ca.pem"
     creds = grpc.ssl_channel_credentials(
-        root_certificates=ca_cert_path.open('rb').read(),
-        private_key=key_path.open('rb').read(),
-        certificate_chain=cert_path.open('rb').read()
+        root_certificates=ca_cert_path.read_bytes(),
+        private_key=key_path.read_bytes(),
+        certificate_chain=cert_path.read_bytes()
     )
 
     wait_for_grpc_start(l1)
@@ -196,15 +196,15 @@ def test_grpc_generate_certificate(node_factory):
     assert [f.exists() for f in files] == [True] * len(files)
 
     # The files exist, restarting should not change them
-    contents = [f.open().read() for f in files]
+    contents = [f.read_bytes() for f in files]
     l1.restart()
-    assert contents == [f.open().read() for f in files]
+    assert contents == [f.read_bytes() for f in files]
 
     # Now we delete the last file, we should regenerate it as well as its key
     files[-1].unlink()
     l1.restart()
-    assert contents[-2] != files[-2].open().read()
-    assert contents[-1] != files[-1].open().read()
+    assert contents[-2] != files[-2].read_bytes()
+    assert contents[-1] != files[-1].read_bytes()
 
     keys = [f for f in files if f.name.endswith('-key.pem')]
     modes = [f.stat().st_mode for f in keys]
@@ -241,7 +241,7 @@ def test_grpc_wrong_auth(node_factory):
 
     def connect(node):
         p = Path(node.daemon.lightning_dir) / TEST_NETWORK
-        cert, key, ca = [f.open('rb').read() for f in [
+        cert, key, ca = [f.read_bytes() for f in [
             p / 'client.pem',
             p / 'client-key.pem',
             p / "ca.pem"]]

tests/test_clnrest.py

@@ -105,22 +105,22 @@ def test_generate_certificate(node_factory):
     wait_for(lambda: [f.exists() for f in files] == [True] * len(files))
 
     # the files exist, restarting should not change them
-    contents = [f.open().read() for f in files]
+    contents = [f.read_bytes() for f in files]
     l1.restart()
-    assert contents == [f.open().read() for f in files]
+    assert contents == [f.read_bytes() for f in files]
 
     # remove client.pem file, so all certs are regenerated at restart
     files[2].unlink()
     l1.restart()
     wait_for(lambda: [f.exists() for f in files] == [True] * len(files))
-    contents_1 = [f.open().read() for f in files]
+    contents_1 = [f.read_bytes() for f in files]
     assert [c[0] != c[1] for c in zip(contents, contents_1)] == [True] * len(files)
 
     # remove client-key.pem file, so all certs are regenerated at restart
     files[3].unlink()
     l1.restart()
     wait_for(lambda: [f.exists() for f in files] == [True] * len(files))
-    contents_2 = [f.open().read() for f in files]
+    contents_2 = [f.read_bytes() for f in files]
     assert [c[0] != c[1] for c in zip(contents, contents_2)] == [True] * len(files)
 
 

tests/test_connection.py

@@ -1,6 +1,7 @@
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
 from decimal import Decimal
+from pathlib import Path
 from pyln.client import RpcError, Millisatoshi
 import pyln.proto.wire as wire
 from utils import (
@@ -3019,7 +3020,7 @@ def test_dataloss_protection(node_factory, bitcoind):
 
     # Save copy of the db.
     dbpath = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "lightningd.sqlite3")
-    orig_db = open(dbpath, "rb").read()
+    orig_db = Path(dbpath).read_bytes()
     l2.start()
 
     # l1 should have sent WIRE_CHANNEL_REESTABLISH with extra fields.
@@ -3063,7 +3064,7 @@ def test_dataloss_protection(node_factory, bitcoind):
     # Now, move l2 back in time.
     l2.stop()
     # Overwrite with OLD db.
-    open(dbpath, "wb").write(orig_db)
+    Path(dbpath).write_bytes(orig_db)
     l2.start()
 
     # l2 should freak out!
@@ -3118,7 +3119,7 @@ def test_dataloss_protection_no_broadcast(node_factory, bitcoind):
 
     # Save copy of the db.
     dbpath = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, "lightningd.sqlite3")
-    orig_db = open(dbpath, "rb").read()
+    orig_db = Path(dbpath).read_bytes()
     l2.start()
 
     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
@@ -3133,9 +3134,9 @@ def test_dataloss_protection_no_broadcast(node_factory, bitcoind):
     # Now, move l2 back in time.
     l2.stop()
     # Save new db
-    new_db = open(dbpath, "rb").read()
+    new_db = Path(dbpath).read_bytes()
     # Overwrite with OLD db.
-    open(dbpath, "wb").write(orig_db)
+    Path(dbpath).write_bytes(orig_db)
     l2.start()
 
     l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
@@ -3149,7 +3150,7 @@ def test_dataloss_protection_no_broadcast(node_factory, bitcoind):
 
     # fix up l2.
     l2.stop()
-    open(dbpath, "wb").write(new_db)
+    Path(dbpath).write_bytes(new_db)
     l2.start()
 
     # All is forgiven