Message-ID: <548F06FE.9090501@oracle.com>
Date:	Mon, 15 Dec 2014 19:06:22 +0300
From:	Alexander Tsvetkov <alexander.tsvetkov@...cle.com>
To:	Andreas Dilger <adilger@...ger.ca>
CC:	fstests@...r.kernel.org, linux-ext4@...r.kernel.org
Subject: Re: max_dir_size_kb option list

Hello Andreas,

Thank you for the review, I've updated test according to your comments

On 12/12/2014 03:12 AM, Andreas Dilger wrote:
> On Dec 11, 2014, at 5:06 AM, Alexander Tsvetkov <alexander.tsvetkov@...cle.com> wrote:
>> Hello,
>>
>> I've prepared test for xfstests suite that runs some checks for ext4 mount option max_dir_size_kb introduced in Linux Kernel 3.7, could someone please look on it?
> For future reference, inline patches as plain text are best.
>
>>  From 21b1da618d0fcb4cd4666d10c41583274ed4eeed Mon Sep 17 00:00:00 2001
>> From: Alexander Tsvetkov <alexander.tsvetkov@...cle.com>
>> Date: Wed, 10 Dec 2014 15:31:02 +0300
>> Subject: [PATCH] added test for max_dir_size_kb mount option
>>
>> ---
>>   tests/ext4/309     | 213 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   tests/ext4/309.out |   2 +
>>   tests/ext4/group   |   3 +-
>>   3 files changed, 217 insertions(+), 1 deletion(-)
>>   create mode 100755 tests/ext4/309
>>   create mode 100755 tests/ext4/309.out
>>
>> diff --git a/tests/ext4/309 b/tests/ext4/309
>> new file mode 100755
>> index 0000000..e2f4e43
>> --- /dev/null
>> +++ b/tests/ext4/309
>> @@ -0,0 +1,213 @@
>> +#! /bin/bash
>> +# FS QA Test
>> +#
>> +# Test for mount option max_dir_size_kb
>> +#
>> +#-----------------------------------------------------------------------
>> +# Copyright (c) 2014 Oracle and/or its affiliates.  All Rights Reserved.
>> +#
>> +# This program is free software; you can redistribute it and/or
>> +# modify it under the terms of the GNU General Public License as
>> +# published by the Free Software Foundation.
>> +#
>> +# This program is distributed in the hope that it would be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write the Free Software Foundation,
>> +# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
>> +#-----------------------------------------------------------------------
>> +#
>> +
>> +seq=`basename $0`
>> +seqres=$RESULT_DIR/$seq
>> +tmp=/tmp/$$
>> +
>> +testdir=$SCRATCH_MNT/testdir
>> +testfile=$SCRATCH_MNT/testfile
>> +sdir=`dirname $0`
>> +sdir=`cd "$sdir"; pwd`
> Since this script is /bin/bash, I'd recommend to use $(...) instead
> of `...` for subshells, since the former can be nested, is much easier
> to see the start/end, and isn't confused for '...' quotes.
>
>> +echo "QA output created by $seq"
>> +echo "Silence is golden"
>> +rm -f $seqres.full
>> +
>> +status=1 # failure is the default!
>> +trap "_cleanup; exit \$status" 0 1 2 3 15
>> +
>> +_cleanup() {
>> +   if [ ! -z SCRATCH_MNT ]; then
>> +      rm -fr $SCRATCH_MNT/test*
>> +      _scratch_unmount
>> +   fi
>> +}
>> +
>> +_filter_error() {
>> +   sed -n -e "s/.*\($1\).*/\"\1\"/p"
>> +}
>> +
>> +_clear_testdir() {
>> +   dirs="$testdir *$"
> This "*$" looks like a bug.  Do you mean "$*"?
>
>> +   for i in $dirs; do
>> +      rm -fr $i/*
>> +   done
>> +}
>> +
>> +# $1 - device
>> +# $2 - options
>> +_make_ext4fs() {
>> +   device=$1
>> +   opts=$2
>> +   umount $device 1>/dev/null 2>&1
>> +   mkfs.ext4 $opts $device 1>/dev/null 2>&1
>> +}
>> +
>> +# $1 - options
>> +# $2 - mount point
>> +_make_loopfs() {
>> +   lpf=$testfile
>> +   dd if=/dev/zero of=$lpf bs=4k count=256 1>/dev/null 2>&1
>> +   loopdev=$(losetup -f)
>> +   losetup $loopdev $lpf
>> +   mkfs.ext4 -O ^dir_index,^has_journal $loopdev 1>/dev/null 2>&1
>> +   mount -t ext4 $1 $loopdev $2
>> +}
>> +
>> +# $1 - expected limit after items creation
>> +# $2 - command to create item
>> +# $3 - where to create (testdir by default)
>> +_create_items() {
>> +    limit=$1
>> +    create_cmd=$2
>> +    dir=${3:-$testdir}
>> +    sync
>> +    echo 3 > /proc/sys/vm/drop_caches
>> +    MAX_INUM=$(((limit*1024*2)/24))
> (style) spaces around operators, and no need for extra () around
> multiplication.
>
>> +    for i in $(seq 0 $MAX_INUM); do
>> +       tmp_name=`mktemp -u`
> Is there a reason to use mktemp instead of just some sequential names
> like item="f.$limit.$i" or similar?  There are several problems with
> the complexity here that could be avoided.
>
> - since mktemp is creating temp files in $TMPDIR by default and "-u"
>    is unlinking them, this is doing double the work needed
> - the "continue" below implies that some items may not be created,
>    due to name collisions (since mktemp doesn't actually see the entries
>    created in the "real" directory), so fewer than $MAX_INUM entries may
>    be created, causing the test to be inconsistent
>
> This could just be something simple like:
>
>           tmp_name=$(mktemp $dir/f.XXXXXX)
>
> and if we need to get directory creation involved, then something like:
>
>           [ "$2" = "mkdir" ] && MKTEMP_OPT="-d"
>           tmp_name=$(mktemp $MKTEMP_OPT $dir/f.XXXXXX 2>$tmp.out)
>
> I see that there are callers of _create_items() that are not "touch"
> or "mkdir", but I don't think those are adding much value to this
> test (more on that below).
>
>> +       item=`basename $tmp_name`
> Use $(...), especially since this style is already used here.
>
applied
>> +       if [ -e $dir/$item ]; then
>> +          continue
>> +       fi
>> +       create_cmd="$2 $dir/$item 2>$tmp.out 1>/dev/null"
>> +       eval "$create_cmd"
>> +       res=$?
>> +       if [ $res -ne 0 ]; then
>> +          _filter_error "No space left on device" < $tmp.out > $tmp.out2
>> +          if [ -s $tmp.out2 ]; then
>> +             cat $tmp.out2 | tr -d '\n' >> $seqres.full
>> +          else
>> +             echo "FAIL! expected ENOSPC" | tee -a $seqres.full
>> +          fi
>> +          break
>> +      fi
>> +   done
>> +   size=$(stat -c %s $dir)
>> +   size=$((size/1024))
>> +   if [ $size -gt $limit ]; then
>> +      echo "FAIL! expected dir size: $limit, actually: $size" | tee -a $seqres.full
>> +   fi
>> +   rm -f $tmp*
>> +}
>> +
>> +# get standard environment, filters and checks
>> +. ./common/rc
>> +
>> +# real QA test starts here
>> +
>> +_supported_fs ext4
>> +_supported_os Linux
>> +_require_scratch
>> +
>> +LIMIT1=8
>> +LIMIT2=16
>> +
>> +_make_ext4fs $SCRATCH_DEV "-O ^dir_index,^filetype"
> It isn't clear why you disable the filetype option?  That has been a
> standard ext2/3/4 feature for 10+ years, so it doesn't make sense to
> test without it.
I tried initially to calculate the number of created items, it was one more
test pass/fail condition, and wanted to be sure that size of created 
dentry item
is based on "ext4_dir_entry" structure. Disabling of dir_index is also 
related to
get more predictable behaviour of number of created items to disable htree
allocations and fill directory up to the maximum items number.At this 
moment
it's clear for me that this check is not required for that test.

>
> It makes sense to test both with and without the dir_index option
> (this is enabled by default for ext4), but it doesn't make sense to
> only test with it disabled.
agree, I've added both test runs with disabled/enabled 'dir_index'
option. Also added one more test with 1Kb growing limits, because
max_dir_size_kb is defined with 1Kb resolution.

>   Better to increase the LIMIT1 and LIMIT2
> values to allow testing with this enabled (e.g. 16 and 32) as well as
> disabled.
added test case with increased limit

>> +_scratch_mount "-o max_dir_size_kb=$LIMIT1"
>> +mkdir $testdir 2>/dev/null
>> +
>> +echo -e "\nExceed $LIMIT1 Kb limit with new files in testdir/: " >> $seqres.full
>> +_create_items $LIMIT1 "touch"
>> +
>> +echo -e "\nRemount with $LIMIT1 Kb limit,\nnew item in testdir/ should result to ENOSPC: " >>$seqres.full
>> +_scratch_mount "-o remount,max_dir_size_kb=$LIMIT1"
>> +_create_items $LIMIT1 "touch"
>> +
>> +echo -e "\nExceed $LIMIT2 Kb limit with new files in testdir/: " >> $seqres.full
>> +_scratch_mount "-o remount,max_dir_size_kb=$LIMIT2"
>> +_create_items $LIMIT2 "touch"
>> +
>> +echo -e "\nExceed $LIMIT2 Kb limit with new files in testdir2/: " >> $seqres.full
>> +mkdir $SCRATCH_MNT/testdir2 2>/dev/null
>> +_create_items $LIMIT2 "touch" "$SCRATCH_MNT/testdir2"
>> +
>> +echo -e "\nRemount with $LIMIT1 Kb limit,\nnew item in testdir/ should result to ENOSPC: " >> $seqres.full
>> +_scratch_mount "-o remount,max_dir_size_kb=$LIMIT1"
>> +_create_items $LIMIT2 "touch"
>> +echo -e "\nnew item in testdir2/ should result to ENOSPC: " >> $seqres.full
>> +_create_items $LIMIT2 "touch" "$SCRATCH_MNT/testdir2"
>> +_clear_testdir "$SCRATCH_MNT/testdir2"
>> +rmdir $testdir
>> +mkdir $testdir
>> +
>> +echo -e "\nExceed $LIMIT1 Kb directory limit with new subdirectories: " >> $seqres.full
>> +_create_items $LIMIT1 "mkdir"
>> +_clear_testdir
>> +
>> +echo -e "\nExceed $LIMIT1 Kb directory limit with symlinks: " >> $seqres.full
>> +dd if=/dev/urandom of=$testfile bs=1 seek=4096 count=4096 > /dev/null 2>&1
>> +_create_items $LIMIT1 "ln -s $testfile"
>> +_clear_testdir
> It isn't clear there is any value to test with symlinks, hard links,
> and FIFOs, since the limit is imposed in the common code for inserting
> the name into the directory, and it doesn't care what type of inode
> the name is referencing.
test was created from "black box" approach, that's why it has some probably
redundant checks, I've removed these meaningless cases.

>> +echo -e "\nExceed $LIMIT1 Kb directory limit with hardlinks: " >> $seqres.full
>> +_create_items $LIMIT1 "ln $testfile"
>> +_clear_testdir
>> +
>> +echo -e "\nExceed $LIMIT1 Kb directory limit with FIFOs: " >> $seqres.full
>> +_create_items $LIMIT1 "mkfifo"
>> +_clear_testdir
>> +
>> +echo -e "\nCreate ext4 fs on testdir/subdir with $LIMIT2 Kb limit," >> $seqres.full
>> +mkdir $testdir/subdir 2>/dev/null
>> +_make_ext4fs $TEST_DEV "-O ^dir_index,^filetype"
>> +mount -t ext4 -o max_dir_size_kb=$LIMIT2 $TEST_DEV $testdir/subdir
>> +
>> +echo "exceed $LIMIT1 Kb limit of testdir/:" >> $seqres.full
>> +_create_items $LIMIT1 "touch"
>> +
>> +echo -e "\nexceed $LIMIT2 Kb limit of testdir/subdir:" >> $seqres.full
>> +_create_items $LIMIT2 "touch" "$testdir/subdir"
>> +
>> +echo -e "\ntestdir/ limit $LIMIT2 Kb, testdir/subdir limit $LIMIT1 Kb," >> $seqres.full
> I think this message is backward. testdir has a limit of $LIMIT1, and
> testdir/subdir has a limit of $LIMIT2.
this is really new test case when directory and subdirectory limits are 
switched,
corrected log message.
>> +umount $TEST_DEV 1>/dev/null 2>&1
>> +_scratch_mount "-o remount,max_dir_size_kb=$LIMIT2"
>> +mount -t ext4 -o max_dir_size_kb=$LIMIT1 $TEST_DEV $testdir/subdir
>> +
>> +echo "exceed new $LIMIT2 Kb limit of testdir/ with a set of files:" >> $seqres.full
>> +_create_items $LIMIT2 "touch"
>> +echo -e "\nnew item in testdir/subdir should result to ENOSPC: " >> $seqres.full
>> +_create_items $LIMIT2 "touch" "$testdir/subdir"
>> +
>> +umount $TEST_DEV 1>/dev/null 2>&1
>> +_clear_testdir
>> +
>> +echo -e "\ntestdir/ limit $LIMIT2 Kb, loop fs: testdir/subdir limit $LIMIT1 Kb," >> $seqres.full
>> +mkdir $testdir/subdir
>> +
>> +_make_loopfs "-o max_dir_size_kb=$LIMIT1" "$testdir/subdir"
>> +
>> +echo "exceed $LIMIT1 Kb limit of testdir/subdir with a set of files:" >> $seqres.full
>> +_create_items $LIMIT1 "touch" "$testdir/subdir"
>> +
>> +echo -e "\nexceed $LIMIT2 Kb limit of testdir/ with a set of files:" >> $seqres.full
>> +_create_items $LIMIT2 "touch"
>> +
>> +umount $testdir/subdir
>> +losetup -d $loopdev
>> +_clear_testdir
>> +
>> +# success, all done
>> +status=0
>> +exit 0
>> diff --git a/tests/ext4/309.out b/tests/ext4/309.out
>> new file mode 100755
>> index 0000000..56330d6
>> --- /dev/null
>> +++ b/tests/ext4/309.out
>> @@ -0,0 +1,2 @@
>> +QA output created by 309
>> +Silence is golden
>> diff --git a/tests/ext4/group b/tests/ext4/group
>> index aa6a53b..9bf1061 100644
>> --- a/tests/ext4/group
>> +++ b/tests/ext4/group
>> @@ -14,4 +14,5 @@
>>   305 auto
>>   306 auto rw resize quick
>>   307 auto ioctl rw
>> -308 auto ioctl rw prealloc quick
>> \ No newline at end of file
>> +308 auto ioctl rw prealloc quick
>> +309 auto
>> -- 
>> 1.9.3
>>
>>
> Cheers, Andreas
>
>
>
>
>

 From e30cd49f5ab84c029c0b376e702caeac42f59f49 Mon Sep 17 00:00:00 2001
From: Alexander Tsvetkov <alexander.tsvetkov@...cle.com>
Date: Mon, 15 Dec 2014 18:49:42 +0300
Subject: [PATCH] added test for max_dir_size_kb mount option

---
  tests/ext4/309     | 178 
+++++++++++++++++++++++++++++++++++++++++++++++++++++
  tests/ext4/309.out |   2 +
  tests/ext4/group   |   1 +
  3 files changed, 181 insertions(+)
  create mode 100755 tests/ext4/309
  create mode 100755 tests/ext4/309.out

diff --git a/tests/ext4/309 b/tests/ext4/309
new file mode 100755
index 0000000..34ceb2b
--- /dev/null
+++ b/tests/ext4/309
@@ -0,0 +1,178 @@
+#! /bin/bash
+# FS QA Test
+#
+# Test for mount option max_dir_size_kb
+#
+#-----------------------------------------------------------------------
+# Copyright (c) 2014 Oracle and/or its affiliates.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#-----------------------------------------------------------------------
+#
+
+seq=$(basename $0)
+seqres=$RESULT_DIR/$seq
+tmp=/tmp/$$
+
+testdir=$SCRATCH_MNT/dir.$seq
+testfile=$SCRATCH_MNT/testfile
+
+echo "QA output created by $seq"
+echo "Silence is golden"
+rm -f $seqres.full
+
+status=1 # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+    cd /
+    rm -rf $tmp.*
+}
+
+remove_files()
+{
+   dirs="$testdir $*"
+   for i in $dirs; do
+      rm -fr $i/*
+   done
+}
+
+# $1 - expected limit after items creation
+# $2 - command to create item
+# $3 - where to create (testdir by default)
+_create_items()
+{
+    limit=$1
+    dir=${2:-$testdir}
+    MKTEMP_OPT=""
+    [ "$3" = "mkdir" ] && MKTEMP_OPT="-d"
+    sync
+    echo 3 > /proc/sys/vm/drop_caches
+    MAX_INUM=$((limit * 1024 * 2 / 24))
+    for i in $(seq 0 $MAX_INUM); do
+       error=$(mktemp $MKTEMP_OPT --tmpdir=$dir 2>&1 >/dev/null)
+       res=$?
+       if [ $res -ne 0 ]; then
+          echo $error >> $seqres.full
+          [[ ! $error =~ ^.*'No space left on device'$ ]] && echo 
"FAIL! expected ENOSPC" | tee -a $seqres.full
+          break
+      fi
+   done
+   size=$(stat -c %s $dir)
+   size=$((size / 1024))
+   if [ $size -gt $limit ]; then
+      echo "FAIL! expected dir size: $limit, actually: $size" | tee -a 
$seqres.full
+   fi
+}
+
+run_test()
+{
+   LIMIT1=$1
+   LIMIT2=$2
+   MKFS_OPT=$3
+
+   _scratch_unmount >/dev/null 2>&1
+   _scratch_mkfs $MKFS_OPT >>$seqres.full 2>&1
+   _scratch_mount -o max_dir_size_kb=$LIMIT1
+   mkdir $testdir
+
+   echo -e "\nExceed $LIMIT1 Kb limit with new files in testdir/: " >> 
$seqres.full
+   _create_items $LIMIT1
+
+   echo -e "\nRemount with $LIMIT1 Kb limit,\nnew item in testdir/ 
should result to ENOSPC: " >>$seqres.full
+   _scratch_mount "-o remount,max_dir_size_kb=$LIMIT1"
+   _create_items $LIMIT1
+
+   echo -e "\nExceed $LIMIT2 Kb limit with new files in testdir/: " >> 
$seqres.full
+   _scratch_mount "-o remount,max_dir_size_kb=$LIMIT2"
+   _create_items $LIMIT2
+
+   echo -e "\nExceed $LIMIT2 Kb limit with new files in testdir2/: " >> 
$seqres.full
+   mkdir $SCRATCH_MNT/testdir2 2>/dev/null
+   _create_items $LIMIT2 "$SCRATCH_MNT/testdir2"
+
+   echo -e "\nRemount with $LIMIT1 Kb limit,\nnew item in testdir/ 
should result to ENOSPC: " >> $seqres.full
+   _scratch_mount "-o remount,max_dir_size_kb=$LIMIT1"
+   _create_items $LIMIT2
+   echo -e "\nnew item in testdir2/ should result to ENOSPC: " >> 
$seqres.full
+   _create_items $LIMIT2 "$SCRATCH_MNT/testdir2"
+   remove_files "$SCRATCH_MNT/testdir2"
+   rmdir $testdir
+   mkdir $testdir
+   dd if=/dev/urandom of=$testfile bs=1 seek=4096 count=4096 > 
/dev/null 2>&1
+
+   echo -e "\nExceed $LIMIT1 Kb directory limit with new 
subdirectories: " >> $seqres.full
+   _create_items $LIMIT1 $testdir "mkdir"
+   remove_files
+
+   echo -e "\nCreate ext4 fs on testdir/subdir with $LIMIT2 Kb limit," 
 >> $seqres.full
+   mkdir $testdir/subdir 2>/dev/null
+   umount $TEST_DEV 1>/dev/null 2>&1
+   _mkfs_dev $TEST_DEV $MKFS_OPT >>$seqres.full 2>&1
+   _mount -o max_dir_size_kb=$LIMIT2 $TEST_DEV $testdir/subdir
+
+   echo "exceed $LIMIT1 Kb limit of testdir/:" >> $seqres.full
+   _create_items $LIMIT1
+
+   echo -e "\nexceed $LIMIT2 Kb limit of testdir/subdir:" >> $seqres.full
+   _create_items $LIMIT2 "$testdir/subdir"
+
+   echo -e "\ntestdir/ limit $LIMIT2 Kb, testdir/subdir limit $LIMIT1 
Kb," >> $seqres.full
+   umount $TEST_DEV 1>/dev/null 2>&1
+   _scratch_mount "-o remount,max_dir_size_kb=$LIMIT2"
+   _mount -o max_dir_size_kb=$LIMIT1 $TEST_DEV $testdir/subdir
+
+   echo "exceed new $LIMIT2 Kb limit of testdir/ with a set of files:" 
 >> $seqres.full
+   _create_items $LIMIT2
+   echo -e "\nnew item in testdir/subdir should result to ENOSPC: " >> 
$seqres.full
+   _create_items $LIMIT2 "$testdir/subdir"
+
+   umount $TEST_DEV 1>/dev/null 2>&1
+   remove_files
+
+   echo -e "\nTestdir/ limit $LIMIT2 Kb, loop fs: testdir/subdir limit 
$LIMIT1 Kb," >> $seqres.full
+   mkdir $testdir/subdir
+
+   $MKFS_EXT4_PROG -F $MKFS_OPT $testfile 2m >> $seqres.full 2>&1
+   _mount -o loop,max_dir_size_kb=$LIMIT1 $testfile $testdir/subdir
+
+   echo "exceed $LIMIT1 Kb limit of testdir/subdir with a set of 
files:" >> $seqres.full
+   _create_items $LIMIT1 "$testdir/subdir"
+
+   echo -e "\nexceed $LIMIT2 Kb limit of testdir/ with a set of files:" 
 >> $seqres.full
+   _create_items $LIMIT2
+
+   umount -d $testdir/subdir
+   remove_files
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+
+# real QA test starts here
+
+_supported_fs ext4
+_supported_os Linux
+_require_scratch
+_require_loop
+
+run_test 8 16
+run_test 4 32 "-O ^dir_index"
+run_test 5 11 "-b 1024"
+
+# success, all done
+status=0
+exit 0
+
diff --git a/tests/ext4/309.out b/tests/ext4/309.out
new file mode 100755
index 0000000..56330d6
--- /dev/null
+++ b/tests/ext4/309.out
@@ -0,0 +1,2 @@
+QA output created by 309
+Silence is golden
diff --git a/tests/ext4/group b/tests/ext4/group
index e7f1f2a..78d643b 100644
--- a/tests/ext4/group
+++ b/tests/ext4/group
@@ -16,3 +16,4 @@
  306 auto rw resize quick
  307 auto ioctl rw
  308 auto ioctl rw prealloc quick
+309 auto
-- 
1.9.3


Thanks,
Alexander Tsvetkov
--
To unsubscribe from this list: send the line "unsubscribe linux-ext4" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Powered by blists - more mailing lists