Newsgroups: php.internals
Path: news.php.net
Xref: news.php.net php.internals:99172
Mailing-List: contact internals-help@lists.php.net; run by ezmlm
Received-SPF: pass (pb1.pair.com: domain lerdorf.com designates 209.85.218.46 as permitted sender)
MIME-Version: 1.0
In-Reply-To: <6F.B1.10292.2B026295@pb1.pair.com>
References: <CALYT1Yrs-9Xr0tyGGWdVBsm0r6h+w+GZcLT0UbVCSMrBYZ01bQ@mail.gmail.com>
 <b62db45d-6b77-f5ec-ac0f-096687d3efe7@fleshgrinder.com> <CADyq6s+=DCk1LiBc3wjUBRdf-mnkgQEqRmGoUeLc+XEjqjk6Qg@mail.gmail.com>
 <1495067758.9756.47.camel@schlueters.de> <CADyq6sKVSgry1AwsqL2EBr96ot35Hoy37pB4ri4wEZeaAWEwiQ@mail.gmail.com>
 <1495103485.9756.58.camel@schlueters.de> <6F.B1.10292.2B026295@pb1.pair.com>
Date: Thu, 25 May 2017 15:34:03 +0300
Message-ID: <CACXBjugP6=EsqXTGT0a7fpE-CzfZ9vyaEyEZ7bhx4EQRkhnj2g@mail.gmail.com>
To: Andrea Faulds <ajf@ajf.me>
Cc: PHP internals <internals@lists.php.net>
Content-Type: multipart/alternative; boundary="001a113ecc8800780405505872bd"
Subject: Re: [PHP-DEV] Implement formal process for run-tests.php refactor
From: rasmus@lerdorf.com (Rasmus Lerdorf)

--001a113ecc8800780405505872bd
Content-Type: text/plain; charset="UTF-8"

On Thu, May 25, 2017 at 3:09 AM, Andrea Faulds <ajf@ajf.me> wrote:
>
> It occurred to me a little while ago that if we ran the test suite in
> parallel, but only run one test from a given directory (or extension) at a
> time, it should probably avoid any such problems, and it would not be
> particularly difficult to implement.
>
> Is that worth a shot, do you think?


Yes, definitely. Some of the tests also have side-effects and don't work if
not run in the original order. They shouldn't have such side-effects, of
course, but auditing nearly 15k tests for this is not a fun task. Within
each directory there should only be one execution thread and the tests
should be run in their original order. That still leaves plenty of room for
concurrency, of course. We have 152 separate directories containing .phpt
files. And if we are smart about it, we sort the directories based on the
number of tests in each and start off with the larger ones to make sure we
don't end with a single process still working on a huge directory. The 152
directories sorted by the number of tests are:

    [./Zend/tests] => 1613
    [./ext/standard/tests/strings] => 924
    [./ext/standard/tests/array] => 923
    [./ext/standard/tests/file] => 800
    [./ext/spl/tests] => 697
    [./ext/date/tests] => 632
    [./ext/intl/tests] => 474
    [./ext/mysqli/tests] => 396
    [./ext/oci8/tests] => 360
    [./ext/reflection/tests] => 334
    [./ext/gd/tests] => 329
    [./ext/mbstring/tests] => 317
    [./ext/phar/tests] => 300
    [./tests/classes] => 277
    [./ext/standard/tests/general_functions] => 272
    [./ext/session/tests] => 228
    [./ext/standard/tests/math] => 219
    [./ext/dom/tests] => 219
    [./tests/lang] => 216
    [./ext/zlib/tests] => 170
    [./ext/pcre/tests] => 118
    [./ext/pdo_mysql/tests] => 116
    [./ext/simplexml/tests] => 112
    [./ext/openssl/tests] => 112
    [./ext/ldap/tests] => 107
    [./Zend/tests/traits] => 107
    [./ext/curl/tests] => 107
    [./ext/phar/tests/tar] => 102
    [./ext/posix/tests] => 100
    [./ext/standard/tests/serialize] => 99
    [./ext/standard/tests/streams] => 96
    [./ext/filter/tests] => 94
    [./ext/phar/tests/zip] => 91
    [./Zend/tests/generators] => 86
    [./ext/soap/tests/schema] => 85
    [./ext/soap/tests/bugs] => 85
    [./ext/xml/tests] => 85
    [./ext/sockets/tests] => 85
    [./ext/sqlite3/tests] => 82
    [./tests/basic] => 80
    [./ext/iconv/tests] => 77
    [./ext/imap/tests] => 77
    [./ext/soap/tests/soap12] => 73
    [./sapi/cli/tests] => 73
    [./ext/soap/tests/interop/Round2/Base] => 72
    [./ext/ctype/tests] => 70
    [./ext/pgsql/tests] => 70
    [./ext/pdo/tests] => 69
    [./ext/standard/tests/class_object] => 68
    [./tests/output] => 66
    [./ext/soap/tests] => 65
    [./ext/mcrypt/tests] => 64
    [./ext/dba/tests] => 64
    [./ext/gmp/tests] => 63
    [./tests/lang/operators] => 62
    [./ext/json/tests] => 62
    [./ext/standard/tests/dir] => 61
    [./ext/xsl/tests] => 61
    [./ext/zip/tests] => 60
    [./ext/phar/tests/cache_list] => 60
    [./ext/soap/tests/interop/Round4/GroupH] => 58
    [./ext/standard/tests/network] => 58
    [./Zend/tests/type_declarations] => 56
    [./ext/opcache/tests] => 56
    [./Zend/tests/return_types] => 54
    [./ext/hash/tests] => 49
    [./ext/exif/tests] => 49
    [./tests/security] => 48
    [./ext/standard/tests/url] => 45
    [./Zend/tests/try] => 43
    [./ext/tidy/tests] => 39
    [./ext/xmlwriter/tests] => 38
    [./ext/fileinfo/tests] => 36
    [./sapi/phpdbg/tests] => 36
    [./ext/soap/tests/interop/Round4/GroupI] => 35
    [./ext/wddx/tests] => 34
    [./ext/pdo_pgsql/tests] => 34
    [./ext/snmp/tests] => 33
    [./ext/bcmath/tests] => 33
    [./ext/xmlrpc/tests] => 32
    [./ext/ftp/tests] => 32
    [./ext/tokenizer/tests] => 32
    [./ext/calendar/tests] => 32
    [./ext/standard/tests/image] => 31
    [./ext/pdo_sqlite/tests] => 28
    [./ext/standard/tests/mail] => 27
    [./ext/pdo_oci/tests] => 27
    [./ext/enchant/tests] => 25
    [./ext/gettext/tests] => 23
    [./Zend/tests/grammar] => 23
    [./ext/xmlreader/tests] => 22
    [./Zend/tests/assert] => 22
    [./sapi/fpm/tests] => 22
    [./ext/interbase/tests] => 22
    [./ext/soap/tests/interop/Round3/GroupD] => 21
    [./ext/standard/tests/assert] => 20
    [./ext/standard/tests/misc] => 20
    [./ext/standard/tests/filters] => 18
    [./ext/libxml/tests] => 18
    [./ext/pcntl/tests] => 18
    [./ext/readline/tests] => 16
    [./Zend/tests/variadic] => 16
    [./Zend/tests/use_function] => 16
    [./ext/soap/tests/interop/Round2/GroupB] => 15
    [./Zend/tests/varSyntax] => 15
    [./ext/pdo_dblib/tests] => 14
    [./ext/standard/tests/http] => 14
    [./Zend/tests/traits/bugs] => 14
    [./sapi/cgi/tests] => 14
    [./tests/func] => 14
    [./tests/run-test] => 14
    [./ext/pdo_firebird/tests] => 14
    [./ext/odbc/tests] => 14
    [./ext/com_dotnet/tests] => 14
    [./Zend/tests/arg_unpack] => 13
    [./Zend/tests/anon] => 12
    [./ext/bz2/tests] => 12
    [./tests/strings] => 11
    [./Zend/tests/use_const] => 11
    [./ext/standard/tests/password] => 10
    [./ext/sysvshm/tests] => 9
    [./tests/lang/string] => 9
    [./Zend/tests/generators/errors] => 9
    [./Zend/tests/generators/finally] => 8
    [./sapi/tests] => 8
    [./ext/standard/tests/time] => 7
    [./ext/standard/tests] => 7
    [./Zend/tests/multibyte] => 7
    [./ext/sysvmsg/tests] => 6
    [./ext/soap/tests/interop/Round3/GroupE] => 6
    [./ext/soap/tests/interop/Round3/GroupF] => 6
    [./ext/pspell/tests] => 5
    [./ext/intl/uchar/tests] => 4
    [./ext/standard/tests/versioning] => 4
    [./Zend/tests/constants] => 4
    [./ext/standard/tests/file/windows_acls] => 4
    [./ext/standard/tests/random] => 4
    [./ext/standard/tests/file/windows_links] => 4
    [./ext/standard/tests/directory] => 4
    [./ext/standard/tests/crypt] => 3
    [./ext/pdo_odbc/tests] => 3
    [./Zend/tests/bug67436] => 2
    [./ext/recode/tests] => 2
    [./Zend/tests/typehints] => 2
    [./tests/lang/constants] => 2
    [./ext/shmop/tests] => 2
    [./ext/sysvsem/tests] => 2
    [./ext/spl/examples/tests] => 1
    [./ext/phar/tests/bug64931] => 1
    [./ext/gmp] => 1
    [./ext/skeleton/tests] => 1
    [./Zend/tests/ast] => 1

--001a113ecc8800780405505872bd--