bigDirTimer

#!/usr/bin/env python3
#
# bigDirTimer.py: Measure how badly big directories slow down.
# 2016-06-21: Written by Steven J. DeRose.
#
import sys
import os
import argparse
from time import time
import shutil

__metadata__ = {
    "title"        : "bigDirTimer",
    "description"  : "Measure how badly big directories slow down.",
    "rightsHolder" : "Steven J. DeRose",
    "creator"      : "http://viaf.org/viaf/50334488",
    "type"         : "http://purl.org/dc/dcmitype/Software",
    "language"     : "Python 3.7",
    "created"      : "2016-06-21",
    "modified"     : "2020-08-23",
    "publisher"    : "http://github.com/sderose",
    "license"      : "https://creativecommons.org/licenses/by-sa/3.0/"
}
__version__ = __metadata__["modified"]


descr = """
=head1 Description

Create a test directory, and fill it with more and more files
(doubling each time) up to I<--max>. Report times to create and access.

"access" can mean either doing a C<listdir()>, or opening and closing
every file, depending on the I<--access> option.

=head1 References

L<http://serverfault.com/questions/43133/filesystem-large-number-of-files-in-a-single-directory> discusses speed issues with large directories for
filesystems such as ext3, ext4, and XFS.

=head1 Known bugs and Limitations

Other than requiring that the directory be newly created on startup,
this script does nothing to avoid caching, garbage collection, optimization,
or confounds from other processes.

=Rights=

Copyright 2016-06-21 by Steven J. DeRose. This work is licensed under a
Creative Commons Attribution-Share-alike 3.0 unported license.
For further information on this license, see
[https://creativecommons.org/licenses/by-sa/3.0].

For the most recent version, see [http://www.derose.net/steve/utilities]
or [https://github.com/sderose].

=head1 Options
"""

###############################################################################
#
def fatal(msg):
    print("\nFATAL ERROR: " + msg)
    sys.exit()
    
def processOptions():
    try:
        from BlockFormatter import BlockFormatter
        parser = argparse.ArgumentParser(
            description=descr, formatter_class=BlockFormatter)
    except ImportError:
        parser = argparse.ArgumentParser(description=descr)

    parser.add_argument(
        "--access", type=str, default="listdir",
        choices=[ "listdir", "openall", ],
        help="How to access the dir once built.")
    parser.add_argument(
        "--data", type=str, default="How big a directory?",
        help="String to write --reps times to each temp file.")
    parser.add_argument(
        "--max", type=int, default=8192,
        help="Max number of files to put in test dir. Default: 8192.")
    parser.add_argument(
        "--quiet", "-q", action="store_true",
        help="Suppress most messages.")
    parser.add_argument(
        "--reps", type=int, default=1,
        help="Number of times to write --data to each temp file. Default: 1.")
    parser.add_argument(
        "--unicode", action="store_const", dest="iencoding",
        const="utf8", help="Assume utf-8 for input files.")
    parser.add_argument(
        "--verbose", "-v", action="count", default=0,
        help="Add more messages (repeatable).")
    parser.add_argument(
        "--version", action="version", version=__version__,
        help="Display version information, then exit.")

    parser.add_argument(
        "files", type=str, nargs=argparse.REMAINDER,
        help="Path(s) to input file(s)")

    args0 = parser.parse_args()
    return(args0)


###############################################################################
# Main
#
args = processOptions()

if (not args.files):
    args.files = [ "/tmp/bigDirTimerData" ]

dirName = args.files[0]
try:
    os.mkdir(dirName)
except OSError as e:
    fatal("Cannot create directory '%s':\n    %s" % (dirName, e))

nFiles = 1


print("Time to build, and then access, n files in 1 directory:")

while (nFiles<=args.max):
    #print("Starting for %d files." % (nFiles))
    startBuild = time()
    for i in range(nFiles):
        f = open(os.path.join(dirName, "temp_%08d" % i), "w", encoding="utf-8")
        f.write(args.data * args.reps)
        f.close()
    endBuild = time()

    startAccess = time()
    if (args.access == "listdir"):
        myList = os.listdir(dirName)
    elif (args.access == "openall"):
        for i in range(nFiles):
            f = open(os.path.join(dirName, "temp_%08d" % nFiles-1-i), "r", encoding="utf-8")
            f.close()
    else:
        fatal("Bad --access value.")
    endAccess = time()
    x = myList[0] + "."

    print("    %8d files,  %12.5fms build, %12.5fms access, %10.5fms / file" %
          (nFiles, (endBuild-startBuild)*1000, (endAccess-startAccess)*1000,
           (endAccess-startAccess)*1000/nFiles))
    nFiles *= 2

if (not args.quiet):
    print("Done. Deleting dir '%s'...." % (dirName))
    shutil.rmtree(dirName)