linux-kernel - Re: Linux 2.6.29

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:	Sat, 4 Apr 2009 18:29:54 +0200
From:	Janne Grunau <j@...nau.net>
To:	Jeff Garzik <jeff@...zik.org>
Cc:	David Rees <drees76@...il.com>, Mark Lord <lkml@....ca>,
	Lennart Sorensen <lsorense@...lub.uwaterloo.ca>,
	Jens Axboe <jens.axboe@...cle.com>,
	Linus Torvalds <torvalds@...ux-foundation.org>,
	Ingo Molnar <mingo@...e.hu>,
	Andrew Morton <akpm@...ux-foundation.org>, tytso@....edu,
	jesper@...gh.cc,
	Linux Kernel Mailing List <linux-kernel@...r.kernel.org>
Subject: Re: Linux 2.6.29

On Fri, Apr 03, 2009 at 07:30:37PM -0400, Jeff Garzik wrote:
> David Rees wrote:
> > The *only* reason MythTV fsyncs (or fdatasyncs) the data to disk all
> > the time is to keep a large amount of dirty pages from building up and
> > then causing horrible latencies when that data starts getting flushed
> > to disk.
> 
> sync_file_range() will definitely help that situation.

Jeff, could you please try following patch for 0.21 or update to the
latest trunk revision. I don't have a way to reproduce the high
latencies with fdatasync on ext3, data=ordered. Doing a parallel
"dd if=/dev/zero of=file" on the same partition introduces even with
sync_file_range latencies over 1 second.

Janne

---

Index: configure
===================================================================
--- configure	(revision 20302)
+++ configure	(working copy)
@@ -873,6 +873,7 @@
     sdl_video_size
     soundcard_h
     stdint_h
+    sync_file_range
     sys_poll_h
     sys_soundcard_h
     termios_h
@@ -2413,6 +2414,17 @@
 int main( void ) { return (round(3.999f) > 0)?0:1; }
 EOF
 
+# test for sync_file_range (linux only system call since 2.6.17)
+check_ld <<EOF && enable sync_file_range
+#define _GNU_SOURCE
+#include <fcntl.h>
+
+int main(int argc, char **argv){
+    sync_file_range(0,0,0,0);
+    return 0;
+}
+EOF
+
 # test for sizeof(int)
 for sizeof in 1 2 4 8 16; do
     check_cc <<EOF && _sizeof_int=$sizeof && break
Index: libs/libmythtv/ThreadedFileWriter.cpp
===================================================================
--- libs/libmythtv/ThreadedFileWriter.cpp	(revision 20302)
+++ libs/libmythtv/ThreadedFileWriter.cpp	(working copy)
@@ -18,6 +18,7 @@
 #include "ThreadedFileWriter.h"
 #include "mythcontext.h"
 #include "compat.h"
+#include "mythconfig.h"
 
 #if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
 #define HAVE_FDATASYNC
@@ -122,6 +123,7 @@
     // file stuff
     filename(QDeepCopy<QString>(fname)), flags(pflags),
     mode(pmode),                         fd(-1),
+    m_file_sync(0),                      m_file_wpos(0),
     // state
     no_writes(false),                    flush(false),
     in_dtor(false),                      ignore_writes(false),
@@ -154,6 +156,8 @@
         buf = new char[TFW_DEF_BUF_SIZE + 1024];
         bzero(buf, TFW_DEF_BUF_SIZE + 64);
 
+        m_file_sync =  m_file_wpos = 0;
+
         tfw_buf_size = TFW_DEF_BUF_SIZE;
         tfw_min_write_size = TFW_MIN_WRITE_SIZE;
         pthread_create(&writer, NULL, boot_writer, this);
@@ -292,7 +296,22 @@
 {
     if (fd >= 0)
     {
-#ifdef HAVE_FDATASYNC
+#ifdef HAVE_SYNC_FILE_RANGE
+        uint64_t write_position;
+
+        buflock.lock();
+        write_position = m_file_wpos;
+        buflock.unlock();
+
+        if ((write_position - m_file_sync) > TFW_MAX_WRITE_SIZE ||
+            (write_position && m_file_sync < (uint64_t)tfw_min_write_size))
+        {
+            sync_file_range(fd, m_file_sync, write_position - m_file_sync,
+                            SYNC_FILE_RANGE_WRITE);
+            m_file_sync = write_position;
+        }
+
+#elif defined(HAVE_FDATASYNC)
         fdatasync(fd);
 #else
         fsync(fd);
@@ -414,6 +433,7 @@
 
         buflock.lock();
         rpos = (rpos + size) % tfw_buf_size;
+        m_file_wpos += size;
         buflock.unlock();
 
         bufferWroteData.wakeAll();
Index: libs/libmythtv/ThreadedFileWriter.h
===================================================================
--- libs/libmythtv/ThreadedFileWriter.h	(revision 20302)
+++ libs/libmythtv/ThreadedFileWriter.h	(working copy)
@@ -40,6 +40,8 @@
     int             flags;
     mode_t          mode;
     int             fd;
+    uint64_t        m_file_sync;  ///< offset synced to disk
+    uint64_t        m_file_wpos; ///< offset written to disk
 
     // state
     bool            no_writes;
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/