| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Wed, 19 Mar 2014 14:16:37 +0100
From: Jan Kara <jack@...e.cz>
To: David Turner <novalis@...alis.org>
Cc: John McCutchan <john@...nmccutchan.com>,
Robert Love <rlove@...ve.org>,
Eric Paris <eparis@...isplace.org>,
linux-kernel@...r.kernel.org
Subject: Re: PROBLEM: Inotify leaks file descriptors.
On Tue 04-03-14 14:18:08, David Turner wrote:
> (script attached)
> On Tue, March 4, 2014 2:09 pm, David Turner wrote:
> > I apologize for the slightly convoluted reproduction steps here,
> > but I was not easily able to find a simpler test case in the
> > time that I had available.
> >
> > First, you'll need Facebook's watchman:
> > https://github.com/facebook/watchman
> >
> > Build and install it. Then run the attached Python script.
> > After a few hundred lines, you'll start to see errors of the form
> > inotify_init error: Too many open files. That could just
> > indicate that watchman is leaking, but I think that's not what's
> > going on, because killing watchman does not fix the problem.
> >
> > To demonstrate, kill the python script, then kill watchman.
> > Then run tail -f /etc/hosts. You'll get "tail: inotify cannot be
> > used, reverting to polling: Too many open files" (you may need to
> > run a few tails to see the error). In fact, the only way I have
> > found to get back to normal is to reboot.
> >
> > I tried increasing the ulimit to 10000 (from the default 1024).
> > The error still happens, but it seems to take a bit longer.
> >
> > I have tried on a couple of Ubuntu kernels:
> >
> > Linux version 3.11.0-17-generic (buildd@...ol) (gcc version 4.6.3
> > (Ubuntu/Linaro 4.6.3-1ubuntu5) ) #31~precise1-Ubuntu SMP Tue Feb 4
> > 21:25:43 UTC 2014
> >
> > And Ubuntu's 3.8.0-36-generic (it's not running right now so I can't give
> > the full version).
> >
> > I've also tried a stock kernel built from source (in a virtualbox):
> >
> > Linux version 3.13.5 (dturner@...rner-virtualbox) (gcc version 4.8.1
> > (Ubuntu/Linaro 4.8.1-10ubuntu8) ) #1 SMP Mon Mar 3 20:41:51 EST 2014
> >
> > I get the error on all of these.
> > There is no output in dmesg.
> >
> > I was running these tests on ext4 filesystems:
> > (for the Ubuntu kernels)
> > /dev/mapper/stross--vg-root on / type ext4 (rw,errors=remount-ro)
> > (for the stock kernel, in the virtualbox)
> > /dev/sda1 on / type ext4 (rw,errors=remount-ro)
> >
> > Please let me know if you need any more information.
> >
> > FWIW, I did find this bug while googling, but it was on older kernels and
> > was allegedly fixed:
> > https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1101666
> >
> >
> >
So when I run your script on 3.14-rc5, I don't see any problems. So maybe
the problem really got fixed? BTW, I'll note that
/proc/sys/fs/inotify/max_user_instances is 128 on my system. Not sure what
it is on your system...
Honza
> #!/usr/bin/python
>
> from atomicinteger import AtomicInteger
> from json import loads, dumps
> from random import random
> from subprocess import call, check_output
> from tempfile import mkdtemp
> from time import sleep, time
>
> import os
> import socket
> import stat
> import threading
>
> #from https://github.com/littlehedgehog/base/blob/master/atomicinteger.py
> class AtomicInteger:
> def __init__(self, integer = 0):
> self.counter = integer
> self.lock = threading.RLock()
> return
>
> def increase(self, inc = 1):
> self.lock.acquire()
> self.counter = self.counter + inc
> self.lock.release()
> return
>
> def decrease(self, dec = 1):
> self.lock.acquire()
> self.counter = self.counter - dec
> self.lock.release()
> return
>
> def get(self):
> return self.counter
>
>
> def get_sockname():
> result = check_output(["watchman", "get-sockname"])
> result = loads(result)
> return result['sockname']
>
> def connect():
> sockname = get_sockname()
> sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
> sock.connect(sockname)
> sock.setblocking(False)
> return sock
>
> def watch(sock, directory):
> watch = ['watch', directory]
> sock.sendall(dumps(watch) + "\n")
> result = readline(sock)
> result = loads(result)
> if not result.get("watch"):
> print result
>
> def readline(sock):
> message = []
> start = time()
> while True:
> elapsed = time() - start
> if elapsed > 5:
> print "We have been waiting a very long time for data from watchman. We have so far: %s" % "".join(message)
> try:
> data = sock.recv(1024, socket.MSG_DONTWAIT)
> if "\n" in data:
> message.append(data[:data.index("\n")])
> break
> except socket.error:
> pass
> sleep(0.001)
> return "".join(message)
>
> def since(sock, directory, since="c:1:2:3:4"):
> expression = {'since' : since}
> watch = ["query", directory, expression]
> sock.sendall(dumps(watch) + "\n")
> message = readline(sock)
> result = loads(message)
> if "error" in result:
> print "Error in since: %s (since = %s)" % (result["error"], since)
> return result
>
> def create(sock, directory=None):
> directory = mkdtemp(dir=directory)
> watch(sock, directory)
> return directory
>
> def touch(directory):
> f = open(os.path.join(directory, "file-%s" % random()), "w")
> f.write("x")
> f.close()
>
> def isdir(f):
> result = os.lstat(f)
> return stat.S_ISDIR(result.st_mode)
>
> def recursive_rmdir(directory):
> for f in os.listdir(directory):
> qualified = os.path.join(directory, f)
> if isdir(qualified):
> recursive_rmdir(qualified)
> else:
> os.unlink(qualified)
> os.rmdir(directory)
>
> nthreads = AtomicInteger()
> runs = AtomicInteger()
>
> def run():
> nthreads.increase()
> runs.increase()
> directory = None
> sock = None
> try:
> print "RUN: %d %d" % (runs.get(), nthreads.get())
> sock = connect()
> directory = create(sock)
>
> result = since(sock, directory)
> assert "files" not in result or len(result["files"]) == 0
> clock = result.get("clock")
> if not clock:
> print "Failed since: %s" % result
> assert clock
>
> #this stanza is only necesary on unbuntu 3.11; on 3.15, it can be skipped
> touch(directory)
> result = since(sock, directory, clock)
> assert result["clock"] != clock
> clock = result["clock"]
> assert len(result["files"]) == 1
>
> sleep(0.1)
>
> #ditto
> for i in range(5):
> touch(directory)
> result = since(sock, directory, clock)
> assert result["clock"] != clock
> if len(result["files"]) < 5:
> print result
>
> finally:
> if sock:
> sock.close()
> if directory:
> recursive_rmdir(directory)
> nthreads.decrease()
>
> def threaded(target, *args, **kwargs):
> thread = threading.Thread(target=target, args=args, kwargs=kwargs)
> thread.start()
>
> while True:
> if nthreads.get() < 15:
> threaded(run)
> else:
> sleep(0.1)
--
Jan Kara <jack@...e.cz>
SUSE Labs, CR
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists